From c13707cd8bba3cac038e730d2f5cdb6421667c83 Mon Sep 17 00:00:00 2001 From: aakcht Date: Fri, 14 Apr 2023 15:48:53 +0300 Subject: [PATCH 1/3] Add more configuration options for helm chart Signed-off-by: aakcht --- installer/README.md | 29 +++++++- .../chart/volcano/templates/admission.yaml | 45 +++++++++++ .../chart/volcano/templates/controllers.yaml | 26 +++++++ .../chart/volcano/templates/scheduler.yaml | 26 +++++++ installer/helm/chart/volcano/values.yaml | 74 +++++++++++++++++++ installer/volcano-development.yaml | 8 ++ 6 files changed, 205 insertions(+), 3 deletions(-) diff --git a/installer/README.md b/installer/README.md index 63dd8c1ea3..acc49732b8 100644 --- a/installer/README.md +++ b/installer/README.md @@ -14,13 +14,10 @@ Kubernetes that are commonly required by many classes of batch & elastic workloa ## Installing volcano via yaml file All-in-one yaml has been generated for quick deployment. Try command: - ```$xslt kubectl apply -f volcano-v0.0.x.yaml ``` - Check the status in namespace `volcano-system` - ```$xslt $kubectl get all -n volcano-system NAME READY STATUS RESTARTS AGE @@ -98,6 +95,31 @@ The following are the list configurable parameters of Volcano Chart and their de |`custom.scheduler_enable`|Whether to Enable Scheduler|`true`| |`custom.scheduler_replicas`|The number of Scheduler pods to run|`1`| |`custom.leader_elect_enable`|Whether to Enable leader elect|`false`| +|`custom.default_affinity`|Default affinity for Admission/Controller/Scheduler pods|`~`| +|`custom.admission_affinity`|Affinity for Admission pods|`~`| +|`custom.controller_affinity`|Affinity for Controller pods|`~`| +|`custom.scheduler_affinity`|Affinity for Scheduler pods|`~`| +|`custom.default_tolerations`|Default tolerations for Admission/Controller/Scheduler pods|`~`| +|`custom.admission_tolerations`|Tolerations for Admission pods|`~`| +|`custom.controller_tolerations`|Tolerations for Controller pods|`~`| +|`custom.scheduler_tolerations`|Tolerations for Scheduler pods|`~`| +|`custom.default_sc`|Default securityContext for Admission/Controller/Scheduler pods|`~`| +|`custom.admission_sc`|securityContext for Admission pods|`~`| +|`custom.controller_sc`|securityContext for Controller pods|`~`| +|`custom.scheduler_sc`|securityContext for Scheduler pods|`~`| +|`custom.default_ns`|Default nodeSelector for Admission/Controller/Scheduler pods|`~`| +|`custom.admission_ns`|nodeSelector for Admission pods|`~`| +|`custom.controller_ns`|nodeSelector for Controller pods|`~`| +|`custom.scheduler_ns`|nodeSelector for Scheduler pods|`~`| +|`custom.admission_podLabels`|Pod labels for Admission pods|`~`| +|`custom.controller_podLabels`|Pod labels for Controller pods|`~`| +|`custom.scheduler_podLabels`|Pod labels for Scheduler pods|`~`| +|`custom.admission_labels`|Labels for Admission deployment and job|`~`| +|`custom.controller_labels`|Labels for Controller deployment|`~`| +|`custom.scheduler_labels`|Labels for Scheduler deployment|`~`| +|`custom.admission_resources`|Resources for Admission pods|`~`| +|`custom.controller_resources`|Resources for Controller pods|`~`| +|`custom.scheduler_resources`|Resources for Scheduler pods|`~`| Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, @@ -107,6 +129,7 @@ $ helm install --name volcano-release --set basic.image_pull_policy=Always volca The above command set image pull policy to `Always`, so docker image will be pulled each time. + Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, ```bash diff --git a/installer/helm/chart/volcano/templates/admission.yaml b/installer/helm/chart/volcano/templates/admission.yaml index 2aa9e5363d..389660e4b3 100644 --- a/installer/helm/chart/volcano/templates/admission.yaml +++ b/installer/helm/chart/volcano/templates/admission.yaml @@ -1,4 +1,8 @@ {{- if .Values.custom.admission_enable }} +{{ $admission_affinity := or .Values.custom.admission_affinity .Values.custom.default_affinity }} +{{ $admission_tolerations := or .Values.custom.admission_tolerations .Values.custom.default_tolerations }} +{{ $admission_sc := or .Values.custom.admission_sc .Values.custom.default_sc }} +{{ $admission_ns := or .Values.custom.admission_ns .Values.custom.default_ns }} apiVersion: v1 kind: ConfigMap metadata: @@ -64,6 +68,9 @@ kind: Deployment metadata: labels: app: volcano-admission + {{- if .Values.custom.admission_labels }} + {{- toYaml .Values.custom.admission_labels | nindent 4 }} + {{- end }} name: {{ .Release.Name }}-admission namespace: {{ .Release.Namespace }} spec: @@ -75,7 +82,24 @@ spec: metadata: labels: app: volcano-admission + {{- if .Values.custom.admission_podLabels }} + {{- toYaml .Values.custom.admission_podLabels | nindent 8 }} + {{- end }} spec: + {{- if $admission_tolerations }} + tolerations: {{- toYaml $admission_tolerations | nindent 8 }} + {{- end }} + {{- if $admission_ns }} + nodeSelector: {{- toYaml $admission_ns | nindent 8 }} + {{- end }} + {{- if $admission_affinity }} + affinity: + {{- toYaml $admission_affinity | nindent 8 }} + {{- end }} + {{- if $admission_sc }} + securityContext: + {{- toYaml $admission_sc | nindent 8 }} + {{- end }} serviceAccount: {{ .Release.Name }}-admission priorityClassName: system-cluster-critical {{- if .Values.basic.image_pull_secret }} @@ -99,6 +123,8 @@ spec: image: {{.Values.basic.admission_image_name}}:{{.Values.basic.image_tag_version}} imagePullPolicy: {{ .Values.basic.image_pull_policy }} name: admission + resources: + {{- toYaml .Values.custom.admission_resources | nindent 12 }} volumeMounts: - mountPath: /admission.local.config/certificates name: admission-certs @@ -152,10 +178,27 @@ metadata: namespace: {{ .Release.Namespace }} labels: app: volcano-admission-init + {{- if .Values.custom.admission_labels }} + {{- toYaml .Values.custom.admission_labels | nindent 4 }} + {{- end }} spec: backoffLimit: 3 template: spec: + {{- if $admission_tolerations }} + tolerations: {{- toYaml $admission_tolerations | nindent 8 }} + {{- end }} + {{- if $admission_ns }} + nodeSelector: {{- toYaml $admission_ns | nindent 8 }} + {{- end }} + {{- if $admission_affinity }} + affinity: + {{- toYaml $admission_affinity | nindent 8 }} + {{- end }} + {{- if $admission_sc }} + securityContext: + {{- toYaml $admission_sc | nindent 8 }} + {{- end }} serviceAccountName: {{ .Release.Name }}-admission priorityClassName: system-cluster-critical {{- if .Values.basic.image_pull_secret }} @@ -165,6 +208,8 @@ spec: restartPolicy: Never containers: - name: main + resources: + {{- toYaml .Values.custom.admission_resources | nindent 12 }} image: {{.Values.basic.admission_image_name}}:{{.Values.basic.image_tag_version}} imagePullPolicy: {{ .Values.basic.image_pull_policy }} command: ["./gen-admission-secret.sh", "--service", "{{ .Release.Name }}-admission-service", "--namespace", diff --git a/installer/helm/chart/volcano/templates/controllers.yaml b/installer/helm/chart/volcano/templates/controllers.yaml index f48c3a0d7e..697bd9f981 100644 --- a/installer/helm/chart/volcano/templates/controllers.yaml +++ b/installer/helm/chart/volcano/templates/controllers.yaml @@ -1,4 +1,8 @@ {{- if .Values.custom.controller_enable }} +{{ $controller_affinity := or .Values.custom.controller_affinity .Values.custom.default_affinity }} +{{ $controller_tolerations := or .Values.custom.controller_tolerations .Values.custom.default_tolerations }} +{{ $controller_sc := or .Values.custom.controller_sc .Values.custom.default_sc }} +{{ $controller_ns := or .Values.custom.controller_ns .Values.custom.default_ns }} apiVersion: v1 kind: ServiceAccount metadata: @@ -90,6 +94,9 @@ metadata: namespace: {{ .Release.Namespace }} labels: app: volcano-controller + {{- if .Values.custom.controller_labels }} + {{- toYaml .Values.custom.controller_labels | nindent 4 }} + {{- end }} spec: replicas: {{ .Values.custom.controller_replicas }} selector: @@ -99,7 +106,24 @@ spec: metadata: labels: app: volcano-controller + {{- if .Values.custom.controller_podLabels }} + {{- toYaml .Values.custom.controller_podLabels | nindent 8 }} + {{- end }} spec: + {{- if $controller_tolerations }} + tolerations: {{- toYaml $controller_tolerations | nindent 8 }} + {{- end }} + {{- if $controller_ns }} + nodeSelector: {{- toYaml $controller_ns | nindent 8 }} + {{- end }} + {{- if $controller_affinity }} + affinity: + {{- toYaml $controller_affinity | nindent 8 }} + {{- end }} + {{- if $controller_sc }} + securityContext: + {{- toYaml $controller_sc | nindent 8 }} + {{- end }} serviceAccount: {{ .Release.Name }}-controllers priorityClassName: system-cluster-critical {{- if .Values.basic.image_pull_secret }} @@ -108,6 +132,8 @@ spec: {{- end }} containers: - name: {{ .Release.Name }}-controllers + resources: + {{- toYaml .Values.custom.controller_resources | nindent 14 }} image: {{.Values.basic.controller_image_name}}:{{.Values.basic.image_tag_version}} args: - --logtostderr diff --git a/installer/helm/chart/volcano/templates/scheduler.yaml b/installer/helm/chart/volcano/templates/scheduler.yaml index a224e6e1e7..756ea5d3d0 100644 --- a/installer/helm/chart/volcano/templates/scheduler.yaml +++ b/installer/helm/chart/volcano/templates/scheduler.yaml @@ -1,4 +1,8 @@ {{- if .Values.custom.scheduler_enable }} +{{ $scheduler_affinity := or .Values.custom.scheduler_affinity .Values.custom.default_affinity }} +{{ $scheduler_tolerations := or .Values.custom.scheduler_tolerations .Values.custom.default_tolerations }} +{{ $scheduler_sc := or .Values.custom.scheduler_sc .Values.custom.default_sc }} +{{ $scheduler_ns := or .Values.custom.scheduler_ns .Values.custom.default_ns }} apiVersion: v1 kind: ConfigMap metadata: @@ -103,6 +107,9 @@ metadata: namespace: {{ .Release.Namespace }} labels: app: volcano-scheduler + {{- if .Values.custom.scheduler_labels }} + {{- toYaml .Values.custom.scheduler_labels | nindent 4 }} + {{- end }} spec: replicas: {{ .Values.custom.scheduler_replicas }} selector: @@ -112,7 +119,24 @@ spec: metadata: labels: app: volcano-scheduler + {{- if .Values.custom.scheduler_podLabels }} + {{- toYaml .Values.custom.scheduler_podLabels | nindent 8 }} + {{- end }} spec: + {{- if $scheduler_tolerations }} + tolerations: {{- toYaml $scheduler_tolerations | nindent 8 }} + {{- end }} + {{- if $scheduler_ns }} + nodeSelector: {{- toYaml $scheduler_ns | nindent 8 }} + {{- end }} + {{- if $scheduler_affinity }} + affinity: + {{- toYaml $scheduler_affinity | nindent 8 }} + {{- end }} + {{- if $scheduler_sc }} + securityContext: + {{- toYaml $scheduler_sc | nindent 8 }} + {{- end }} serviceAccount: {{ .Release.Name }}-scheduler priorityClassName: system-cluster-critical {{- if .Values.basic.image_pull_secret }} @@ -122,6 +146,8 @@ spec: containers: - name: {{ .Release.Name }}-scheduler image: {{.Values.basic.scheduler_image_name}}:{{.Values.basic.image_tag_version}} + resources: + {{- toYaml .Values.custom.scheduler_resources | nindent 12 }} args: - --logtostderr - --scheduler-conf=/volcano.scheduler/{{base .Values.basic.scheduler_config_file}} diff --git a/installer/helm/chart/volcano/values.yaml b/installer/helm/chart/volcano/values.yaml index 0b29d9780a..901259bb41 100644 --- a/installer/helm/chart/volcano/values.yaml +++ b/installer/helm/chart/volcano/values.yaml @@ -22,3 +22,77 @@ custom: nodeSelector: {} affinity: [] tolerations: [] + +# Specify affinity for all main Volcano components or per component. +# For example: +# +# default_affinity: +# podAntiAffinity: +# preferredDuringSchedulingIgnoredDuringExecution: +# - podAffinityTerm: +# labelSelector: +# matchLabels: +# key: value +# topologyKey: kubernetes.io/hostname +# weight: 49 + default_affinity: ~ + admission_affinity: ~ + controller_affinity: ~ + scheduler_affinity: ~ + +# Specify tolerations for all main Volcano components or per component +# For example: +# +# default_tolerations: +# - key: "example-key1" +# operator: "Exists" +# effect: "NoSchedule" + default_tolerations: ~ + admission_tolerations: ~ + controller_tolerations: ~ + scheduler_tolerations: ~ + +# Specify securityContext for all main Volcano components or per component +# For example: +# +# default_sc: +# runAsUser: 3000 +# runAsGroup: 3000 + default_sc: ~ + scheduler_sc: ~ + admission_sc: ~ + controller_sc: ~ + +# Specify nodeSelector for all main Volcano components or per component +# For example: +# +# default_ns: +# nodetype: criticalservices + default_ns: ~ + admission_ns: ~ + scheduler_ns: ~ + controller_ns: ~ + + +# Specify labels for Volcano main component deployments and pods +# For example: +# +# admission_podLabels: +# key1: value1 + admission_podLabels: ~ + scheduler_podLabels: ~ + controller_podLabels: ~ + admission_labels: ~ + scheduler_labels: ~ + controller_labels: ~ + +# Specify resources for Volcano main component deployments and pods +# For example: +# +# admission_resources: +# limits: +# cpu: 300m +# memory: 300Mi + admission_resources: ~ + scheduler_resources: ~ + controller_resources: ~ diff --git a/installer/volcano-development.yaml b/installer/volcano-development.yaml index 6e2fcbe68a..dd617ca65a 100644 --- a/installer/volcano-development.yaml +++ b/installer/volcano-development.yaml @@ -153,6 +153,8 @@ spec: image: volcanosh/vc-webhook-manager:latest imagePullPolicy: Always name: admission + resources: + null volumeMounts: - mountPath: /admission.local.config/certificates name: admission-certs @@ -185,6 +187,8 @@ spec: restartPolicy: Never containers: - name: main + resources: + null image: volcanosh/vc-webhook-manager:latest imagePullPolicy: Always command: ["./gen-admission-secret.sh", "--service", "volcano-admission-service", "--namespace", @@ -8616,6 +8620,8 @@ spec: priorityClassName: system-cluster-critical containers: - name: volcano-controllers + resources: + null image: volcanosh/vc-controller-manager:latest args: - --logtostderr @@ -8785,6 +8791,8 @@ spec: containers: - name: volcano-scheduler image: volcanosh/vc-scheduler:latest + resources: + null args: - --logtostderr - --scheduler-conf=/volcano.scheduler/volcano-scheduler.conf From cc765e8b00c5b7ef525af6106aafd396dd0d5d78 Mon Sep 17 00:00:00 2001 From: aakcht Date: Tue, 23 May 2023 07:27:12 +0300 Subject: [PATCH 2/3] resolve merge conflicts Signed-off-by: aakcht --- installer/README.md | 4 +++- .../helm/chart/volcano/templates/admission.yaml | 13 ------------- .../helm/chart/volcano/templates/controllers.yaml | 12 ------------ .../helm/chart/volcano/templates/scheduler.yaml | 12 ------------ installer/helm/chart/volcano/values.yaml | 3 --- 5 files changed, 3 insertions(+), 41 deletions(-) diff --git a/installer/README.md b/installer/README.md index acc49732b8..41c176e407 100644 --- a/installer/README.md +++ b/installer/README.md @@ -14,10 +14,13 @@ Kubernetes that are commonly required by many classes of batch & elastic workloa ## Installing volcano via yaml file All-in-one yaml has been generated for quick deployment. Try command: + ```$xslt kubectl apply -f volcano-v0.0.x.yaml ``` + Check the status in namespace `volcano-system` + ```$xslt $kubectl get all -n volcano-system NAME READY STATUS RESTARTS AGE @@ -129,7 +132,6 @@ $ helm install --name volcano-release --set basic.image_pull_policy=Always volca The above command set image pull policy to `Always`, so docker image will be pulled each time. - Alternatively, a YAML file that specifies the values for the parameters can be provided while installing the chart. For example, ```bash diff --git a/installer/helm/chart/volcano/templates/admission.yaml b/installer/helm/chart/volcano/templates/admission.yaml index 389660e4b3..ba627b16ab 100644 --- a/installer/helm/chart/volcano/templates/admission.yaml +++ b/installer/helm/chart/volcano/templates/admission.yaml @@ -140,19 +140,6 @@ spec: configMap: name: {{ .Release.Name }}-admission-configmap - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - --- apiVersion: v1 kind: Service diff --git a/installer/helm/chart/volcano/templates/controllers.yaml b/installer/helm/chart/volcano/templates/controllers.yaml index 697bd9f981..da30b86c90 100644 --- a/installer/helm/chart/volcano/templates/controllers.yaml +++ b/installer/helm/chart/volcano/templates/controllers.yaml @@ -145,16 +145,4 @@ spec: - -v=4 - 2>&1 imagePullPolicy: {{ .Values.basic.image_pull_policy }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} {{- end }} diff --git a/installer/helm/chart/volcano/templates/scheduler.yaml b/installer/helm/chart/volcano/templates/scheduler.yaml index 756ea5d3d0..f377635e60 100644 --- a/installer/helm/chart/volcano/templates/scheduler.yaml +++ b/installer/helm/chart/volcano/templates/scheduler.yaml @@ -167,18 +167,6 @@ spec: - name: scheduler-config configMap: name: {{ .Release.Name }}-scheduler-configmap - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} --- apiVersion: v1 kind: Service diff --git a/installer/helm/chart/volcano/values.yaml b/installer/helm/chart/volcano/values.yaml index 901259bb41..c679b94f5b 100644 --- a/installer/helm/chart/volcano/values.yaml +++ b/installer/helm/chart/volcano/values.yaml @@ -19,9 +19,6 @@ custom: scheduler_replicas: 1 leader_elect_enable: false enabled_admissions: "/jobs/mutate,/jobs/validate,/podgroups/mutate,/pods/validate,/pods/mutate,/queues/mutate,/queues/validate" -nodeSelector: {} -affinity: [] -tolerations: [] # Specify affinity for all main Volcano components or per component. # For example: From eb1f1c49211e5643a9876cea4293fbeb93fe3d98 Mon Sep 17 00:00:00 2001 From: aakcht Date: Fri, 16 Jun 2023 15:03:40 +0300 Subject: [PATCH 3/3] avoid null resources in volcano-development.yaml Signed-off-by: aakcht --- installer/helm/chart/volcano/templates/admission.yaml | 4 ++++ installer/helm/chart/volcano/templates/controllers.yaml | 2 ++ installer/helm/chart/volcano/templates/scheduler.yaml | 2 ++ installer/volcano-development.yaml | 8 -------- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/installer/helm/chart/volcano/templates/admission.yaml b/installer/helm/chart/volcano/templates/admission.yaml index ba627b16ab..8cfdce4648 100644 --- a/installer/helm/chart/volcano/templates/admission.yaml +++ b/installer/helm/chart/volcano/templates/admission.yaml @@ -123,8 +123,10 @@ spec: image: {{.Values.basic.admission_image_name}}:{{.Values.basic.image_tag_version}} imagePullPolicy: {{ .Values.basic.image_pull_policy }} name: admission + {{- if .Values.custom.admission_resources }} resources: {{- toYaml .Values.custom.admission_resources | nindent 12 }} + {{- end }} volumeMounts: - mountPath: /admission.local.config/certificates name: admission-certs @@ -195,8 +197,10 @@ spec: restartPolicy: Never containers: - name: main + {{- if .Values.custom.admission_resources }} resources: {{- toYaml .Values.custom.admission_resources | nindent 12 }} + {{- end }} image: {{.Values.basic.admission_image_name}}:{{.Values.basic.image_tag_version}} imagePullPolicy: {{ .Values.basic.image_pull_policy }} command: ["./gen-admission-secret.sh", "--service", "{{ .Release.Name }}-admission-service", "--namespace", diff --git a/installer/helm/chart/volcano/templates/controllers.yaml b/installer/helm/chart/volcano/templates/controllers.yaml index da30b86c90..7c2929df9e 100644 --- a/installer/helm/chart/volcano/templates/controllers.yaml +++ b/installer/helm/chart/volcano/templates/controllers.yaml @@ -132,8 +132,10 @@ spec: {{- end }} containers: - name: {{ .Release.Name }}-controllers + {{- if .Values.custom.controller_resources }} resources: {{- toYaml .Values.custom.controller_resources | nindent 14 }} + {{- end }} image: {{.Values.basic.controller_image_name}}:{{.Values.basic.image_tag_version}} args: - --logtostderr diff --git a/installer/helm/chart/volcano/templates/scheduler.yaml b/installer/helm/chart/volcano/templates/scheduler.yaml index f377635e60..98636f17bf 100644 --- a/installer/helm/chart/volcano/templates/scheduler.yaml +++ b/installer/helm/chart/volcano/templates/scheduler.yaml @@ -146,8 +146,10 @@ spec: containers: - name: {{ .Release.Name }}-scheduler image: {{.Values.basic.scheduler_image_name}}:{{.Values.basic.image_tag_version}} + {{- if .Values.custom.scheduler_resources }} resources: {{- toYaml .Values.custom.scheduler_resources | nindent 12 }} + {{- end }} args: - --logtostderr - --scheduler-conf=/volcano.scheduler/{{base .Values.basic.scheduler_config_file}} diff --git a/installer/volcano-development.yaml b/installer/volcano-development.yaml index dd617ca65a..6e2fcbe68a 100644 --- a/installer/volcano-development.yaml +++ b/installer/volcano-development.yaml @@ -153,8 +153,6 @@ spec: image: volcanosh/vc-webhook-manager:latest imagePullPolicy: Always name: admission - resources: - null volumeMounts: - mountPath: /admission.local.config/certificates name: admission-certs @@ -187,8 +185,6 @@ spec: restartPolicy: Never containers: - name: main - resources: - null image: volcanosh/vc-webhook-manager:latest imagePullPolicy: Always command: ["./gen-admission-secret.sh", "--service", "volcano-admission-service", "--namespace", @@ -8620,8 +8616,6 @@ spec: priorityClassName: system-cluster-critical containers: - name: volcano-controllers - resources: - null image: volcanosh/vc-controller-manager:latest args: - --logtostderr @@ -8791,8 +8785,6 @@ spec: containers: - name: volcano-scheduler image: volcanosh/vc-scheduler:latest - resources: - null args: - --logtostderr - --scheduler-conf=/volcano.scheduler/volcano-scheduler.conf