enable https healthz listen;

Signed-off-by: shaoqiu <516595344@qq.com>

cmd/controller-manager/app/options/options.go

@@ -18,6 +18,7 @@ package options
 
 import (
 	"fmt"
+	"io/ioutil"
 	"time"
 
 	"github.com/spf13/pflag"
@@ -38,6 +39,10 @@ const (
 // ServerOption is the main context object for the controllers.
 type ServerOption struct {
 	KubeClientOptions    kube.ClientOptions
+	CertFile             string
+	KeyFile              string
+	CertData             []byte
+	KeyData              []byte
 	EnableLeaderElection bool
 	LockObjectNamespace  string
 	PrintVersion         bool
@@ -59,6 +64,8 @@ type ServerOption struct {
 	DetectionPeriodOfDependsOntask time.Duration
 }
 
+type DecryptFunc func(c *ServerOption) error
+
 // NewServerOption creates a new CMServer with a default config.
 func NewServerOption() *ServerOption {
 	return &ServerOption{}
@@ -68,6 +75,10 @@ func NewServerOption() *ServerOption {
 func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.KubeClientOptions.Master, "master", s.KubeClientOptions.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
 	fs.StringVar(&s.KubeClientOptions.KubeConfig, "kubeconfig", s.KubeClientOptions.KubeConfig, "Path to kubeconfig file with authorization and master location information.")
+	fs.StringVar(&s.CertFile, "tls-cert-file", s.CertFile, ""+
+		"File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+
+		"after server cert).")
+	fs.StringVar(&s.KeyFile, "tls-private-key-file", s.KeyFile, "File containing the default x509 private key matching --tls-cert-file.")
 	fs.BoolVar(&s.EnableLeaderElection, "leader-elect", s.EnableLeaderElection, "Start a leader election client and gain leadership before "+
 		"executing the main loop. Enable this when running replicated vc-controller-manager for high availability.")
 	fs.StringVar(&s.LockObjectNamespace, "lock-object-namespace", s.LockObjectNamespace, "Define the namespace of the lock object.")
@@ -91,3 +102,34 @@ func (s *ServerOption) CheckOptionOrDie() error {
 	}
 	return nil
 }
+
+// readCAFiles read data from ca file path
+func (s *ServerOption) readCAFiles() error {
+	var err error
+
+	s.CertData, err = ioutil.ReadFile(s.CertFile)
+	if err != nil {
+		return fmt.Errorf("failed to read cert file (%s): %v", s.CertFile, err)
+	}
+
+	s.KeyData, err = ioutil.ReadFile(s.KeyFile)
+	if err != nil {
+		return fmt.Errorf("failed to read key file (%s): %v", s.KeyFile, err)
+	}
+
+	return nil
+}
+
+// ParseCAFiles parse ca file by decryptFunc
+func (s *ServerOption) ParseCAFiles(decryptFunc DecryptFunc) error {
+	if err := s.readCAFiles(); err != nil {
+		return err
+	}
+
+	// users can add one function to decrypt tha data by their own way if CA data is encrypted
+	if decryptFunc != nil {
+		return decryptFunc(s)
+	}
+
+	return nil
+}

cmd/controller-manager/app/server.go

@@ -56,7 +56,7 @@ func Run(opt *options.ServerOption) error {
 	}
 
 	if opt.EnableHealthz {
-		if err := helpers.StartHealthz(opt.HealthzBindAddress, "volcano-controller"); err != nil {
+		if err := helpers.StartHealthz(opt.HealthzBindAddress, "volcano-controller", opt.CertData, opt.KeyData); err != nil {
 			return err
 		}
 	}

cmd/controller-manager/main.go

@@ -56,6 +56,13 @@ func main() {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		os.Exit(1)
 	}
+	if s.CertFile != "" && s.KeyFile != "" {
+		if err := s.ParseCAFiles(nil); err != nil {
+			fmt.Fprintf(os.Stderr, "Failed to parse CA file: %v\n", err)
+			os.Exit(1)
+		}
+	}
+
 	// The default klog flush interval is 30 seconds, which is frighteningly long.
 	go wait.Until(klog.Flush, *logFlushFreq, wait.NeverStop)
 	defer klog.Flush()

cmd/scheduler/app/options/options.go

@@ -18,6 +18,7 @@ package options
 
 import (
 	"fmt"
+	"io/ioutil"
 	"time"
 
 	"github.com/spf13/pflag"
@@ -45,6 +46,10 @@ const (
 // ServerOption is the main context object for the controller manager.
 type ServerOption struct {
 	KubeClientOptions    kube.ClientOptions
+	CertFile             string
+	KeyFile              string
+	CertData             []byte
+	KeyData              []byte
 	SchedulerNames       []string
 	SchedulerConf        string
 	SchedulePeriod       time.Duration
@@ -70,6 +75,8 @@ type ServerOption struct {
 	NodeSelector []string
 }
 
+type DecryptFunc func(c *ServerOption) error
+
 // ServerOpts server options.
 var ServerOpts *ServerOption
 
@@ -82,6 +89,10 @@ func NewServerOption() *ServerOption {
 func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.KubeClientOptions.Master, "master", s.KubeClientOptions.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
 	fs.StringVar(&s.KubeClientOptions.KubeConfig, "kubeconfig", s.KubeClientOptions.KubeConfig, "Path to kubeconfig file with authorization and master location information")
+	fs.StringVar(&s.CertFile, "tls-cert-file", s.CertFile, ""+
+		"File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+
+		"after server cert).")
+	fs.StringVar(&s.KeyFile, "tls-private-key-file", s.KeyFile, "File containing the default x509 private key matching --tls-cert-file.")
 	// volcano scheduler will ignore pods with scheduler names other than specified with the option
 	fs.StringArrayVar(&s.SchedulerNames, "scheduler-name", []string{defaultSchedulerName}, "vc-scheduler will handle pods whose .spec.SchedulerName is same as scheduler-name")
 	fs.StringVar(&s.SchedulerConf, "scheduler-conf", "", "The absolute path of scheduler configuration file")
@@ -129,3 +140,34 @@ func (s *ServerOption) CheckOptionOrDie() error {
 func (s *ServerOption) RegisterOptions() {
 	ServerOpts = s
 }
+
+// readCAFiles read data from ca file path
+func (s *ServerOption) readCAFiles() error {
+	var err error
+
+	s.CertData, err = ioutil.ReadFile(s.CertFile)
+	if err != nil {
+		return fmt.Errorf("failed to read cert file (%s): %v", s.CertFile, err)
+	}
+
+	s.KeyData, err = ioutil.ReadFile(s.KeyFile)
+	if err != nil {
+		return fmt.Errorf("failed to read key file (%s): %v", s.KeyFile, err)
+	}
+
+	return nil
+}
+
+// ParseCAFiles parse ca file by decryptFunc
+func (s *ServerOption) ParseCAFiles(decryptFunc DecryptFunc) error {
+	if err := s.readCAFiles(); err != nil {
+		return err
+	}
+
+	// users can add one function to decrypt tha data by their own way if CA data is encrypted
+	if decryptFunc != nil {
+		return decryptFunc(s)
+	}
+
+	return nil
+}

cmd/scheduler/app/server.go

@@ -92,7 +92,7 @@ func Run(opt *options.ServerOption) error {
 	}
 
 	if opt.EnableHealthz {
-		if err := helpers.StartHealthz(opt.HealthzBindAddress, "volcano-scheduler"); err != nil {
+		if err := helpers.StartHealthz(opt.HealthzBindAddress, "volcano-scheduler", opt.CertData, opt.KeyData); err != nil {
 			return err
 		}
 	}

cmd/scheduler/main.go

@@ -59,6 +59,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	if s.CertFile != "" && s.KeyFile != "" {
+		if err := s.ParseCAFiles(nil); err != nil {
+			klog.Fatalf("Failed to parse CA file: %v", err)
+		}
+	}
+
 	go wait.Until(klog.Flush, *logFlushFreq, wait.NeverStop)
 	defer klog.Flush()
 

config/crd/bases/scheduling.volcano.sh_queues.yaml

@@ -123,6 +123,9 @@ spec:
                 description: Reclaimable indicate whether the queue can be reclaimed
                   by other queue
                 type: boolean
+              type:
+                description: Type define the type of queue
+                type: string
               weight:
                 format: int32
                 type: integer

config/crd/v1beta1/scheduling.volcano.sh_queues.yaml

@@ -122,6 +122,9 @@ spec:
               description: Reclaimable indicate whether the queue can be reclaimed
                 by other queue
               type: boolean
+            type:
+              description: Type define the type of queue
+              type: string
             weight:
               format: int32
               type: integer

go.mod

@@ -32,7 +32,7 @@ require (
 	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b
 	sigs.k8s.io/yaml v1.3.0
 	stathat.com/c/consistent v1.0.0
-	volcano.sh/apis v1.6.0-alpha.0.0.20220712043845-8d8aa5aecbd2
+	volcano.sh/apis v1.6.0-alpha.0.0.20221012070524-685db38b4fae
 )
 
 require (

go.sum

@@ -1201,5 +1201,5 @@ sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
 stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c=
 stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0=
-volcano.sh/apis v1.6.0-alpha.0.0.20220712043845-8d8aa5aecbd2 h1:8p4FIUbVepYoyxMKxnb6W8PohzweIrIh06YvCHklq78=
-volcano.sh/apis v1.6.0-alpha.0.0.20220712043845-8d8aa5aecbd2/go.mod h1:drNMGuHPn1ew7oBSDQb5KRey6tXOQksbUtw3gPxF3Vo=
+volcano.sh/apis v1.6.0-alpha.0.0.20221012070524-685db38b4fae h1:H7yidKnIq/Y7KmjFP5xFSmE7xL674226D8pEoA/RfG8=
+volcano.sh/apis v1.6.0-alpha.0.0.20221012070524-685db38b4fae/go.mod h1:drNMGuHPn1ew7oBSDQb5KRey6tXOQksbUtw3gPxF3Vo=

installer/helm/chart/volcano/crd/bases/scheduling.volcano.sh_queues.yaml

@@ -121,6 +121,9 @@ spec:
                 description: Reclaimable indicate whether the queue can be reclaimed
                   by other queue
                 type: boolean
+              type:
+                description: Type define the type of queue
+                type: string
               weight:
                 format: int32
                 type: integer

installer/helm/chart/volcano/crd/v1beta1/scheduling.volcano.sh_queues.yaml

@@ -120,6 +120,9 @@ spec:
               description: Reclaimable indicate whether the queue can be reclaimed
                 by other queue
               type: boolean
+            type:
+              description: Type define the type of queue
+              type: string
             weight:
               format: int32
               type: integer

installer/volcano-development.yaml

@@ -8954,6 +8954,9 @@ spec:
                 description: Reclaimable indicate whether the queue can be reclaimed
                   by other queue
                 type: boolean
+              type:
+                description: Type define the type of queue
+                type: string
               weight:
                 format: int32
                 type: integer

vendor/modules.txt

@@ -1050,7 +1050,7 @@ sigs.k8s.io/yaml
 # stathat.com/c/consistent v1.0.0
 ## explicit
 stathat.com/c/consistent
-# volcano.sh/apis v1.6.0-alpha.0.0.20220712043845-8d8aa5aecbd2
+# volcano.sh/apis v1.6.0-alpha.0.0.20221012070524-685db38b4fae
 ## explicit; go 1.17
 volcano.sh/apis/pkg/apis/batch/v1alpha1
 volcano.sh/apis/pkg/apis/bus/v1alpha1

vendor/volcano.sh/apis/pkg/apis/helpers/helpers.go

@@ -18,6 +18,7 @@ package helpers
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"net"
 	"net/http"
@@ -188,7 +189,7 @@ func GeneratePodgroupName(pod *v1.Pod) string {
 }
 
 // StartHealthz register healthz interface.
-func StartHealthz(healthzBindAddress, name string) error {
+func StartHealthz(healthzBindAddress, name string, certData, certKeyData []byte) error {
 	listener, err := net.Listen("tcp", healthzBindAddress)
 	if err != nil {
 		return fmt.Errorf("failed to create listener: %v", err)
@@ -202,6 +203,15 @@ func StartHealthz(healthzBindAddress, name string) error {
 		Handler:        pathRecorderMux,
 		MaxHeaderBytes: 1 << 20,
 	}
+	if len(certData) != 0 && len(certKeyData) != 0 {
+		sCert, err := tls.X509KeyPair(certData, certKeyData)
+		if err != nil {
+			return fmt.Errorf("failed to parse certData: %v", err)
+		}
+		server.TLSConfig = &tls.Config{
+			Certificates: []tls.Certificate{sCert},
+		}
+	}
 
 	return runServer(server, listener)
 }
@@ -226,7 +236,12 @@ func runServer(server *http.Server, ln net.Listener) error {
 
 		listener := tcpKeepAliveListener{ln.(*net.TCPListener)}
 
-		err := server.Serve(listener)
+		var err error
+		if server.TLSConfig != nil {
+			err = server.ServeTLS(listener, "", "")
+		} else {
+			err = server.Serve(listener)
+		}
 		msg := fmt.Sprintf("Stopped listening on %s", listener.Addr().String())
 		select {
 		case <-stopCh:

vendor/volcano.sh/apis/pkg/apis/scheduling/types.go

@@ -295,7 +295,6 @@ type Cluster struct {
 	Capacity v1.ResourceList
 }
 
-
 // Affinity is a group of affinity scheduling rules.
 type Affinity struct {
 	// Describes nodegroup affinity scheduling rules for the queue.
@@ -309,16 +308,16 @@ type Affinity struct {
 
 type NodeGroupAffinity struct {
 	// +optional
-	RequiredDuringSchedulingIgnoredDuringExecution  []string `json:"requiredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,1,opt,name=requiredDuringSchedulingIgnoredDuringExecution"`
+	RequiredDuringSchedulingIgnoredDuringExecution []string `json:"requiredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,1,opt,name=requiredDuringSchedulingIgnoredDuringExecution"`
 	// +optional
-	PreferredDuringSchedulingIgnoredDuringExecution  []string `json:"preferredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,2,rep,name=preferredDuringSchedulingIgnoredDuringExecution"`
+	PreferredDuringSchedulingIgnoredDuringExecution []string `json:"preferredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,2,rep,name=preferredDuringSchedulingIgnoredDuringExecution"`
 }
 
 type NodeGroupAntiAffinity struct {
 	// +optional
-	RequiredDuringSchedulingIgnoredDuringExecution  []string `json:"requiredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,1,opt,name=requiredDuringSchedulingIgnoredDuringExecution"`
+	RequiredDuringSchedulingIgnoredDuringExecution []string `json:"requiredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,1,opt,name=requiredDuringSchedulingIgnoredDuringExecution"`
 	// +optional
-	PreferredDuringSchedulingIgnoredDuringExecution  []string `json:"preferredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,2,rep,name=preferredDuringSchedulingIgnoredDuringExecution"`
+	PreferredDuringSchedulingIgnoredDuringExecution []string `json:"preferredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,2,rep,name=preferredDuringSchedulingIgnoredDuringExecution"`
 }
 
 // QueueSpec represents the template of Queue.
@@ -340,6 +339,9 @@ type QueueSpec struct {
 	// If specified, the queue's scheduling constraints
 	// +optional
 	Affinity *Affinity `json:"affinity,omitempty" protobuf:"bytes,6,opt,name=affinity"`
+
+	// Type define the type of queue
+	Type string `json:"type,omitempty" protobuf:"bytes,7,opt,name=type"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/labels.go

@@ -35,6 +35,10 @@ const KubeGroupNameAnnotationKey = "scheduling.k8s.io/group-name"
 // which PodGroup it belongs to.
 const VolcanoGroupNameAnnotationKey = GroupName + "/group-name"
 
+// VolcanoGroupMinResourcesAnnotationKey is the annotation key of PodGroup's PodGroup.Spec.MinResources
+// which PodGroup it belongs to.
+const VolcanoGroupMinResourcesAnnotationKey = GroupName + "/group-min-resources"
+
 // QueueNameAnnotationKey is the annotation key of Pod to identify
 // which queue it belongs to.
 const QueueNameAnnotationKey = GroupName + "/queue-name"

vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/types.go

@@ -353,6 +353,9 @@ type QueueSpec struct {
 	// If specified, the pod owned by the queue will be scheduled with constraint
 	// +optional
 	Affinity *Affinity `json:"affinity,omitempty" protobuf:"bytes,6,opt,name=affinity"`
+
+	// Type define the type of queue
+	Type string `json:"type,omitempty" protobuf:"bytes,7,opt,name=type"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object