diff --git a/cmd/controller-manager/app/options/options.go b/cmd/controller-manager/app/options/options.go index f8586288c57..24600222524 100644 --- a/cmd/controller-manager/app/options/options.go +++ b/cmd/controller-manager/app/options/options.go @@ -18,6 +18,7 @@ package options import ( "fmt" + "io/ioutil" "time" "github.com/spf13/pflag" @@ -38,6 +39,10 @@ const ( // ServerOption is the main context object for the controllers. type ServerOption struct { KubeClientOptions kube.ClientOptions + CertFile string + KeyFile string + CertData []byte + KeyData []byte EnableLeaderElection bool LockObjectNamespace string PrintVersion bool @@ -59,6 +64,8 @@ type ServerOption struct { DetectionPeriodOfDependsOntask time.Duration } +type DecryptFunc func(c *ServerOption) error + // NewServerOption creates a new CMServer with a default config. func NewServerOption() *ServerOption { return &ServerOption{} @@ -68,6 +75,10 @@ func NewServerOption() *ServerOption { func (s *ServerOption) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.KubeClientOptions.Master, "master", s.KubeClientOptions.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") fs.StringVar(&s.KubeClientOptions.KubeConfig, "kubeconfig", s.KubeClientOptions.KubeConfig, "Path to kubeconfig file with authorization and master location information.") + fs.StringVar(&s.CertFile, "tls-cert-file", s.CertFile, ""+ + "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+ + "after server cert).") + fs.StringVar(&s.KeyFile, "tls-private-key-file", s.KeyFile, "File containing the default x509 private key matching --tls-cert-file.") fs.BoolVar(&s.EnableLeaderElection, "leader-elect", s.EnableLeaderElection, "Start a leader election client and gain leadership before "+ "executing the main loop. Enable this when running replicated vc-controller-manager for high availability.") fs.StringVar(&s.LockObjectNamespace, "lock-object-namespace", s.LockObjectNamespace, "Define the namespace of the lock object.") @@ -91,3 +102,34 @@ func (s *ServerOption) CheckOptionOrDie() error { } return nil } + +// readCAFiles read data from ca file path +func (s *ServerOption) readCAFiles() error { + var err error + + s.CertData, err = ioutil.ReadFile(s.CertFile) + if err != nil { + return fmt.Errorf("failed to read cert file (%s): %v", s.CertFile, err) + } + + s.KeyData, err = ioutil.ReadFile(s.KeyFile) + if err != nil { + return fmt.Errorf("failed to read key file (%s): %v", s.KeyFile, err) + } + + return nil +} + +// ParseCAFiles parse ca file by decryptFunc +func (s *ServerOption) ParseCAFiles(decryptFunc DecryptFunc) error { + if err := s.readCAFiles(); err != nil { + return err + } + + // users can add one function to decrypt tha data by their own way if CA data is encrypted + if decryptFunc != nil { + return decryptFunc(s) + } + + return nil +} diff --git a/cmd/controller-manager/app/server.go b/cmd/controller-manager/app/server.go index 34747dc9a65..1d7bd36d013 100644 --- a/cmd/controller-manager/app/server.go +++ b/cmd/controller-manager/app/server.go @@ -56,7 +56,7 @@ func Run(opt *options.ServerOption) error { } if opt.EnableHealthz { - if err := helpers.StartHealthz(opt.HealthzBindAddress, "volcano-controller"); err != nil { + if err := helpers.StartHealthz(opt.HealthzBindAddress, "volcano-controller", opt.CertData, opt.KeyData); err != nil { return err } } diff --git a/cmd/controller-manager/main.go b/cmd/controller-manager/main.go index cc3627f2a10..5cecc0f4c93 100644 --- a/cmd/controller-manager/main.go +++ b/cmd/controller-manager/main.go @@ -56,6 +56,13 @@ func main() { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } + if s.CertFile != "" && s.KeyFile != "" { + if err := s.ParseCAFiles(nil); err != nil { + fmt.Fprintf(os.Stderr, "Failed to parse CA file: %v\n", err) + os.Exit(1) + } + } + // The default klog flush interval is 30 seconds, which is frighteningly long. go wait.Until(klog.Flush, *logFlushFreq, wait.NeverStop) defer klog.Flush() diff --git a/cmd/scheduler/app/options/options.go b/cmd/scheduler/app/options/options.go index 4720c62e643..73d93ee4995 100644 --- a/cmd/scheduler/app/options/options.go +++ b/cmd/scheduler/app/options/options.go @@ -18,6 +18,7 @@ package options import ( "fmt" + "io/ioutil" "time" "github.com/spf13/pflag" @@ -45,6 +46,10 @@ const ( // ServerOption is the main context object for the controller manager. type ServerOption struct { KubeClientOptions kube.ClientOptions + CertFile string + KeyFile string + CertData []byte + KeyData []byte SchedulerNames []string SchedulerConf string SchedulePeriod time.Duration @@ -70,6 +75,8 @@ type ServerOption struct { NodeSelector []string } +type DecryptFunc func(c *ServerOption) error + // ServerOpts server options. var ServerOpts *ServerOption @@ -82,6 +89,10 @@ func NewServerOption() *ServerOption { func (s *ServerOption) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&s.KubeClientOptions.Master, "master", s.KubeClientOptions.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)") fs.StringVar(&s.KubeClientOptions.KubeConfig, "kubeconfig", s.KubeClientOptions.KubeConfig, "Path to kubeconfig file with authorization and master location information") + fs.StringVar(&s.CertFile, "tls-cert-file", s.CertFile, ""+ + "File containing the default x509 Certificate for HTTPS. (CA cert, if any, concatenated "+ + "after server cert).") + fs.StringVar(&s.KeyFile, "tls-private-key-file", s.KeyFile, "File containing the default x509 private key matching --tls-cert-file.") // volcano scheduler will ignore pods with scheduler names other than specified with the option fs.StringArrayVar(&s.SchedulerNames, "scheduler-name", []string{defaultSchedulerName}, "vc-scheduler will handle pods whose .spec.SchedulerName is same as scheduler-name") fs.StringVar(&s.SchedulerConf, "scheduler-conf", "", "The absolute path of scheduler configuration file") @@ -129,3 +140,34 @@ func (s *ServerOption) CheckOptionOrDie() error { func (s *ServerOption) RegisterOptions() { ServerOpts = s } + +// readCAFiles read data from ca file path +func (s *ServerOption) readCAFiles() error { + var err error + + s.CertData, err = ioutil.ReadFile(s.CertFile) + if err != nil { + return fmt.Errorf("failed to read cert file (%s): %v", s.CertFile, err) + } + + s.KeyData, err = ioutil.ReadFile(s.KeyFile) + if err != nil { + return fmt.Errorf("failed to read key file (%s): %v", s.KeyFile, err) + } + + return nil +} + +// ParseCAFiles parse ca file by decryptFunc +func (s *ServerOption) ParseCAFiles(decryptFunc DecryptFunc) error { + if err := s.readCAFiles(); err != nil { + return err + } + + // users can add one function to decrypt tha data by their own way if CA data is encrypted + if decryptFunc != nil { + return decryptFunc(s) + } + + return nil +} diff --git a/cmd/scheduler/app/server.go b/cmd/scheduler/app/server.go index 746a9e16b9a..6a8712bf865 100644 --- a/cmd/scheduler/app/server.go +++ b/cmd/scheduler/app/server.go @@ -92,7 +92,7 @@ func Run(opt *options.ServerOption) error { } if opt.EnableHealthz { - if err := helpers.StartHealthz(opt.HealthzBindAddress, "volcano-scheduler"); err != nil { + if err := helpers.StartHealthz(opt.HealthzBindAddress, "volcano-scheduler", opt.CertData, opt.KeyData); err != nil { return err } } diff --git a/cmd/scheduler/main.go b/cmd/scheduler/main.go index b8d07258ae2..02249458178 100644 --- a/cmd/scheduler/main.go +++ b/cmd/scheduler/main.go @@ -59,6 +59,12 @@ func main() { os.Exit(1) } + if s.CertFile != "" && s.KeyFile != "" { + if err := s.ParseCAFiles(nil); err != nil { + klog.Fatalf("Failed to parse CA file: %v", err) + } + } + go wait.Until(klog.Flush, *logFlushFreq, wait.NeverStop) defer klog.Flush() diff --git a/config/crd/bases/scheduling.volcano.sh_queues.yaml b/config/crd/bases/scheduling.volcano.sh_queues.yaml index 4609ba8d922..7a1f2474f94 100644 --- a/config/crd/bases/scheduling.volcano.sh_queues.yaml +++ b/config/crd/bases/scheduling.volcano.sh_queues.yaml @@ -123,6 +123,9 @@ spec: description: Reclaimable indicate whether the queue can be reclaimed by other queue type: boolean + type: + description: Type define the type of queue + type: string weight: format: int32 type: integer diff --git a/config/crd/v1beta1/scheduling.volcano.sh_queues.yaml b/config/crd/v1beta1/scheduling.volcano.sh_queues.yaml index e1fdf7dcb57..edc22cb2e77 100644 --- a/config/crd/v1beta1/scheduling.volcano.sh_queues.yaml +++ b/config/crd/v1beta1/scheduling.volcano.sh_queues.yaml @@ -122,6 +122,9 @@ spec: description: Reclaimable indicate whether the queue can be reclaimed by other queue type: boolean + type: + description: Type define the type of queue + type: string weight: format: int32 type: integer diff --git a/go.mod b/go.mod index 25d99fce23f..9c2d4180c2d 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b sigs.k8s.io/yaml v1.3.0 stathat.com/c/consistent v1.0.0 - volcano.sh/apis v1.6.0-alpha.0.0.20220712043845-8d8aa5aecbd2 + volcano.sh/apis v1.6.0-alpha.0.0.20221012070524-685db38b4fae ) require ( diff --git a/go.sum b/go.sum index 2e0019c93a4..c3a053fd6aa 100644 --- a/go.sum +++ b/go.sum @@ -1201,5 +1201,5 @@ sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= -volcano.sh/apis v1.6.0-alpha.0.0.20220712043845-8d8aa5aecbd2 h1:8p4FIUbVepYoyxMKxnb6W8PohzweIrIh06YvCHklq78= -volcano.sh/apis v1.6.0-alpha.0.0.20220712043845-8d8aa5aecbd2/go.mod h1:drNMGuHPn1ew7oBSDQb5KRey6tXOQksbUtw3gPxF3Vo= +volcano.sh/apis v1.6.0-alpha.0.0.20221012070524-685db38b4fae h1:H7yidKnIq/Y7KmjFP5xFSmE7xL674226D8pEoA/RfG8= +volcano.sh/apis v1.6.0-alpha.0.0.20221012070524-685db38b4fae/go.mod h1:drNMGuHPn1ew7oBSDQb5KRey6tXOQksbUtw3gPxF3Vo= diff --git a/installer/helm/chart/volcano/crd/bases/scheduling.volcano.sh_queues.yaml b/installer/helm/chart/volcano/crd/bases/scheduling.volcano.sh_queues.yaml index 1d417da0617..1e21b021038 100644 --- a/installer/helm/chart/volcano/crd/bases/scheduling.volcano.sh_queues.yaml +++ b/installer/helm/chart/volcano/crd/bases/scheduling.volcano.sh_queues.yaml @@ -121,6 +121,9 @@ spec: description: Reclaimable indicate whether the queue can be reclaimed by other queue type: boolean + type: + description: Type define the type of queue + type: string weight: format: int32 type: integer diff --git a/installer/helm/chart/volcano/crd/v1beta1/scheduling.volcano.sh_queues.yaml b/installer/helm/chart/volcano/crd/v1beta1/scheduling.volcano.sh_queues.yaml index b422f3c9de0..b07530f1c3d 100644 --- a/installer/helm/chart/volcano/crd/v1beta1/scheduling.volcano.sh_queues.yaml +++ b/installer/helm/chart/volcano/crd/v1beta1/scheduling.volcano.sh_queues.yaml @@ -120,6 +120,9 @@ spec: description: Reclaimable indicate whether the queue can be reclaimed by other queue type: boolean + type: + description: Type define the type of queue + type: string weight: format: int32 type: integer diff --git a/installer/volcano-development.yaml b/installer/volcano-development.yaml index 1dda9064a0a..264c71fcab4 100644 --- a/installer/volcano-development.yaml +++ b/installer/volcano-development.yaml @@ -8954,6 +8954,9 @@ spec: description: Reclaimable indicate whether the queue can be reclaimed by other queue type: boolean + type: + description: Type define the type of queue + type: string weight: format: int32 type: integer diff --git a/vendor/modules.txt b/vendor/modules.txt index a46fb9f71ab..38ada7951ae 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1050,7 +1050,7 @@ sigs.k8s.io/yaml # stathat.com/c/consistent v1.0.0 ## explicit stathat.com/c/consistent -# volcano.sh/apis v1.6.0-alpha.0.0.20220712043845-8d8aa5aecbd2 +# volcano.sh/apis v1.6.0-alpha.0.0.20221012070524-685db38b4fae ## explicit; go 1.17 volcano.sh/apis/pkg/apis/batch/v1alpha1 volcano.sh/apis/pkg/apis/bus/v1alpha1 diff --git a/vendor/volcano.sh/apis/pkg/apis/helpers/helpers.go b/vendor/volcano.sh/apis/pkg/apis/helpers/helpers.go index df21f62bfbc..aa14ca5e231 100644 --- a/vendor/volcano.sh/apis/pkg/apis/helpers/helpers.go +++ b/vendor/volcano.sh/apis/pkg/apis/helpers/helpers.go @@ -18,6 +18,7 @@ package helpers import ( "context" + "crypto/tls" "fmt" "net" "net/http" @@ -188,7 +189,7 @@ func GeneratePodgroupName(pod *v1.Pod) string { } // StartHealthz register healthz interface. -func StartHealthz(healthzBindAddress, name string) error { +func StartHealthz(healthzBindAddress, name string, certData, certKeyData []byte) error { listener, err := net.Listen("tcp", healthzBindAddress) if err != nil { return fmt.Errorf("failed to create listener: %v", err) @@ -202,6 +203,15 @@ func StartHealthz(healthzBindAddress, name string) error { Handler: pathRecorderMux, MaxHeaderBytes: 1 << 20, } + if len(certData) != 0 && len(certKeyData) != 0 { + sCert, err := tls.X509KeyPair(certData, certKeyData) + if err != nil { + return fmt.Errorf("failed to parse certData: %v", err) + } + server.TLSConfig = &tls.Config{ + Certificates: []tls.Certificate{sCert}, + } + } return runServer(server, listener) } @@ -226,7 +236,12 @@ func runServer(server *http.Server, ln net.Listener) error { listener := tcpKeepAliveListener{ln.(*net.TCPListener)} - err := server.Serve(listener) + var err error + if server.TLSConfig != nil { + err = server.ServeTLS(listener, "", "") + } else { + err = server.Serve(listener) + } msg := fmt.Sprintf("Stopped listening on %s", listener.Addr().String()) select { case <-stopCh: diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/types.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/types.go index 35d2b08f6b1..508cc337217 100644 --- a/vendor/volcano.sh/apis/pkg/apis/scheduling/types.go +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/types.go @@ -295,7 +295,6 @@ type Cluster struct { Capacity v1.ResourceList } - // Affinity is a group of affinity scheduling rules. type Affinity struct { // Describes nodegroup affinity scheduling rules for the queue. @@ -309,16 +308,16 @@ type Affinity struct { type NodeGroupAffinity struct { // +optional - RequiredDuringSchedulingIgnoredDuringExecution []string `json:"requiredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,1,opt,name=requiredDuringSchedulingIgnoredDuringExecution"` + RequiredDuringSchedulingIgnoredDuringExecution []string `json:"requiredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,1,opt,name=requiredDuringSchedulingIgnoredDuringExecution"` // +optional - PreferredDuringSchedulingIgnoredDuringExecution []string `json:"preferredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,2,rep,name=preferredDuringSchedulingIgnoredDuringExecution"` + PreferredDuringSchedulingIgnoredDuringExecution []string `json:"preferredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,2,rep,name=preferredDuringSchedulingIgnoredDuringExecution"` } type NodeGroupAntiAffinity struct { // +optional - RequiredDuringSchedulingIgnoredDuringExecution []string `json:"requiredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,1,opt,name=requiredDuringSchedulingIgnoredDuringExecution"` + RequiredDuringSchedulingIgnoredDuringExecution []string `json:"requiredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,1,opt,name=requiredDuringSchedulingIgnoredDuringExecution"` // +optional - PreferredDuringSchedulingIgnoredDuringExecution []string `json:"preferredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,2,rep,name=preferredDuringSchedulingIgnoredDuringExecution"` + PreferredDuringSchedulingIgnoredDuringExecution []string `json:"preferredDuringSchedulingIgnoredDuringExecution,omitempty" protobuf:"bytes,2,rep,name=preferredDuringSchedulingIgnoredDuringExecution"` } // QueueSpec represents the template of Queue. @@ -340,6 +339,9 @@ type QueueSpec struct { // If specified, the queue's scheduling constraints // +optional Affinity *Affinity `json:"affinity,omitempty" protobuf:"bytes,6,opt,name=affinity"` + + // Type define the type of queue + Type string `json:"type,omitempty" protobuf:"bytes,7,opt,name=type"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/labels.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/labels.go index 4a4d793f2c8..eb923a33dc3 100644 --- a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/labels.go +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/labels.go @@ -35,6 +35,10 @@ const KubeGroupNameAnnotationKey = "scheduling.k8s.io/group-name" // which PodGroup it belongs to. const VolcanoGroupNameAnnotationKey = GroupName + "/group-name" +// VolcanoGroupMinResourcesAnnotationKey is the annotation key of PodGroup's PodGroup.Spec.MinResources +// which PodGroup it belongs to. +const VolcanoGroupMinResourcesAnnotationKey = GroupName + "/group-min-resources" + // QueueNameAnnotationKey is the annotation key of Pod to identify // which queue it belongs to. const QueueNameAnnotationKey = GroupName + "/queue-name" diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/types.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/types.go index 6c053ab09b1..82abba681d9 100644 --- a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/types.go +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/types.go @@ -353,6 +353,9 @@ type QueueSpec struct { // If specified, the pod owned by the queue will be scheduled with constraint // +optional Affinity *Affinity `json:"affinity,omitempty" protobuf:"bytes,6,opt,name=affinity"` + + // Type define the type of queue + Type string `json:"type,omitempty" protobuf:"bytes,7,opt,name=type"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.conversion.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.conversion.go index dd8b4b47fbd..7562fe6cd09 100644 --- a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.conversion.go +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.conversion.go @@ -522,6 +522,7 @@ func autoConvert_v1beta1_QueueSpec_To_scheduling_QueueSpec(in *QueueSpec, out *s return err } out.Affinity = (*scheduling.Affinity)(unsafe.Pointer(in.Affinity)) + out.Type = in.Type return nil } @@ -540,6 +541,7 @@ func autoConvert_scheduling_QueueSpec_To_v1beta1_QueueSpec(in *scheduling.QueueS return err } out.Affinity = (*Affinity)(unsafe.Pointer(in.Affinity)) + out.Type = in.Type return nil }