Merge pull request #2788 from Aakcht/helm_more_install_options

Add more configuration options for helm chart
volcano-sh · Jun 28, 2023 · 92646d2 · 92646d2
2 parents c83fd01 + eb1f1c4
commit 92646d2
Show file tree

Hide file tree

Showing 5 changed files with 204 additions and 40 deletions.
diff --git a/installer/README.md b/installer/README.md
@@ -98,6 +98,31 @@ The following are the list configurable parameters of Volcano Chart and their de
 |`custom.scheduler_enable`|Whether to Enable Scheduler|`true`|
 |`custom.scheduler_replicas`|The number of Scheduler pods to run|`1`|
 |`custom.leader_elect_enable`|Whether to Enable leader elect|`false`|
+|`custom.default_affinity`|Default affinity for Admission/Controller/Scheduler pods|`~`|
+|`custom.admission_affinity`|Affinity for Admission pods|`~`|
+|`custom.controller_affinity`|Affinity for Controller pods|`~`|
+|`custom.scheduler_affinity`|Affinity for Scheduler pods|`~`|
+|`custom.default_tolerations`|Default tolerations for Admission/Controller/Scheduler pods|`~`|
+|`custom.admission_tolerations`|Tolerations for Admission pods|`~`|
+|`custom.controller_tolerations`|Tolerations for Controller pods|`~`|
+|`custom.scheduler_tolerations`|Tolerations for Scheduler pods|`~`|
+|`custom.default_sc`|Default securityContext for Admission/Controller/Scheduler pods|`~`|
+|`custom.admission_sc`|securityContext for Admission pods|`~`|
+|`custom.controller_sc`|securityContext for Controller pods|`~`|
+|`custom.scheduler_sc`|securityContext for Scheduler pods|`~`|
+|`custom.default_ns`|Default nodeSelector for Admission/Controller/Scheduler pods|`~`|
+|`custom.admission_ns`|nodeSelector for Admission pods|`~`|
+|`custom.controller_ns`|nodeSelector for Controller pods|`~`|
+|`custom.scheduler_ns`|nodeSelector for Scheduler pods|`~`|
+|`custom.admission_podLabels`|Pod labels for Admission pods|`~`|
+|`custom.controller_podLabels`|Pod labels for Controller pods|`~`|
+|`custom.scheduler_podLabels`|Pod labels for Scheduler pods|`~`|
+|`custom.admission_labels`|Labels for Admission deployment and job|`~`|
+|`custom.controller_labels`|Labels for Controller deployment|`~`|
+|`custom.scheduler_labels`|Labels for Scheduler deployment|`~`|
+|`custom.admission_resources`|Resources for Admission pods|`~`|
+|`custom.controller_resources`|Resources for Controller pods|`~`|
+|`custom.scheduler_resources`|Resources for Scheduler pods|`~`|
 
 Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example,
 

diff --git a/installer/helm/chart/volcano/templates/admission.yaml b/installer/helm/chart/volcano/templates/admission.yaml
@@ -1,4 +1,8 @@
 {{- if .Values.custom.admission_enable }}
+{{ $admission_affinity := or .Values.custom.admission_affinity .Values.custom.default_affinity }}
+{{ $admission_tolerations := or .Values.custom.admission_tolerations .Values.custom.default_tolerations }}
+{{ $admission_sc := or .Values.custom.admission_sc .Values.custom.default_sc }}
+{{ $admission_ns := or .Values.custom.admission_ns .Values.custom.default_ns }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -64,6 +68,9 @@ kind: Deployment
 metadata:
   labels:
     app: volcano-admission
+    {{- if .Values.custom.admission_labels }}
+    {{- toYaml .Values.custom.admission_labels | nindent 4 }}
+    {{- end }}
   name: {{ .Release.Name }}-admission
   namespace: {{ .Release.Namespace }}
 spec:
@@ -75,7 +82,24 @@ spec:
     metadata:
       labels:
         app: volcano-admission
+        {{- if .Values.custom.admission_podLabels }}
+        {{- toYaml .Values.custom.admission_podLabels | nindent 8 }}
+        {{- end }}
     spec:
+      {{- if $admission_tolerations }}
+      tolerations: {{- toYaml $admission_tolerations | nindent 8 }}
+      {{- end }}
+      {{- if $admission_ns }}
+      nodeSelector: {{- toYaml $admission_ns | nindent 8 }}
+      {{- end }}
+      {{- if $admission_affinity }}
+      affinity:
+          {{- toYaml $admission_affinity | nindent 8 }}
+      {{- end }}
+      {{- if $admission_sc }}
+      securityContext:
+          {{- toYaml $admission_sc | nindent 8 }}
+      {{- end }}
       serviceAccount: {{ .Release.Name }}-admission
       priorityClassName: system-cluster-critical
       {{- if .Values.basic.image_pull_secret }}
@@ -99,6 +123,10 @@ spec:
           image: {{.Values.basic.admission_image_name}}:{{.Values.basic.image_tag_version}}
           imagePullPolicy: {{ .Values.basic.image_pull_policy }}
           name: admission
+          {{- if .Values.custom.admission_resources }}
+          resources:
+          {{- toYaml .Values.custom.admission_resources | nindent 12 }}
+          {{- end }}
           volumeMounts:
             - mountPath: /admission.local.config/certificates
               name: admission-certs
@@ -114,19 +142,6 @@ spec:
           configMap:
             name: {{ .Release.Name }}-admission-configmap
 
-      {{- with .Values.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-
 ---
 apiVersion: v1
 kind: Service
@@ -152,10 +167,27 @@ metadata:
   namespace: {{ .Release.Namespace }}
   labels:
     app: volcano-admission-init
+    {{- if .Values.custom.admission_labels }}
+    {{- toYaml .Values.custom.admission_labels | nindent 4 }}
+    {{- end }}
 spec:
   backoffLimit: 3
   template:
     spec:
+      {{- if $admission_tolerations }}
+      tolerations: {{- toYaml $admission_tolerations | nindent 8 }}
+      {{- end }}
+      {{- if $admission_ns }}
+      nodeSelector: {{- toYaml $admission_ns | nindent 8 }}
+      {{- end }}
+      {{- if $admission_affinity }}
+      affinity:
+        {{- toYaml $admission_affinity | nindent 8 }}
+      {{- end }}
+      {{- if $admission_sc }}
+      securityContext:
+        {{- toYaml $admission_sc | nindent 8 }}
+      {{- end }}
       serviceAccountName: {{ .Release.Name }}-admission
       priorityClassName: system-cluster-critical
       {{- if .Values.basic.image_pull_secret }}
@@ -165,6 +197,10 @@ spec:
       restartPolicy: Never
       containers:
         - name: main
+          {{- if .Values.custom.admission_resources }}
+          resources:
+          {{- toYaml .Values.custom.admission_resources | nindent 12 }}
+          {{- end }}
           image: {{.Values.basic.admission_image_name}}:{{.Values.basic.image_tag_version}}
           imagePullPolicy: {{ .Values.basic.image_pull_policy }}
           command: ["./gen-admission-secret.sh", "--service", "{{ .Release.Name }}-admission-service", "--namespace",

diff --git a/installer/helm/chart/volcano/templates/controllers.yaml b/installer/helm/chart/volcano/templates/controllers.yaml
@@ -1,4 +1,8 @@
 {{- if .Values.custom.controller_enable }}
+{{ $controller_affinity := or .Values.custom.controller_affinity .Values.custom.default_affinity }}
+{{ $controller_tolerations := or .Values.custom.controller_tolerations .Values.custom.default_tolerations }}
+{{ $controller_sc := or .Values.custom.controller_sc .Values.custom.default_sc }}
+{{ $controller_ns := or .Values.custom.controller_ns .Values.custom.default_ns }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -90,6 +94,9 @@ metadata:
   namespace: {{ .Release.Namespace }}
   labels:
     app: volcano-controller
+    {{- if .Values.custom.controller_labels }}
+    {{- toYaml .Values.custom.controller_labels | nindent 4 }}
+    {{- end }}
 spec:
   replicas: {{ .Values.custom.controller_replicas }}
   selector:
@@ -99,7 +106,24 @@ spec:
     metadata:
       labels:
         app: volcano-controller
+        {{- if .Values.custom.controller_podLabels }}
+        {{- toYaml .Values.custom.controller_podLabels | nindent 8 }}
+        {{- end }}
     spec:
+      {{- if $controller_tolerations }}
+      tolerations: {{- toYaml $controller_tolerations | nindent 8 }}
+      {{- end }}
+      {{- if $controller_ns }}
+      nodeSelector: {{- toYaml $controller_ns | nindent 8 }}
+      {{- end }}
+      {{- if $controller_affinity }}
+      affinity:
+        {{- toYaml $controller_affinity | nindent 8 }}
+      {{- end }}
+      {{- if $controller_sc }}
+      securityContext:
+        {{- toYaml $controller_sc | nindent 8 }}
+      {{- end }}
       serviceAccount: {{ .Release.Name }}-controllers
       priorityClassName: system-cluster-critical
       {{- if .Values.basic.image_pull_secret }}
@@ -108,6 +132,10 @@ spec:
       {{- end }}
       containers:
           - name: {{ .Release.Name }}-controllers
+            {{- if .Values.custom.controller_resources }}
+            resources:
+            {{- toYaml .Values.custom.controller_resources | nindent 14 }}
+            {{- end }}
             image: {{.Values.basic.controller_image_name}}:{{.Values.basic.image_tag_version}}
             args:
               - --logtostderr
@@ -119,16 +147,4 @@ spec:
               - -v=4
               - 2>&1
             imagePullPolicy: {{ .Values.basic.image_pull_policy }}
-      {{- with .Values.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
 {{- end }}
diff --git a/installer/helm/chart/volcano/templates/scheduler.yaml b/installer/helm/chart/volcano/templates/scheduler.yaml
@@ -1,4 +1,8 @@
 {{- if .Values.custom.scheduler_enable }}
+{{ $scheduler_affinity := or .Values.custom.scheduler_affinity .Values.custom.default_affinity }}
+{{ $scheduler_tolerations := or .Values.custom.scheduler_tolerations .Values.custom.default_tolerations }}
+{{ $scheduler_sc := or .Values.custom.scheduler_sc .Values.custom.default_sc }}
+{{ $scheduler_ns := or .Values.custom.scheduler_ns .Values.custom.default_ns }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -103,6 +107,9 @@ metadata:
   namespace: {{ .Release.Namespace }}
   labels:
     app: volcano-scheduler
+    {{- if .Values.custom.scheduler_labels }}
+    {{- toYaml .Values.custom.scheduler_labels | nindent 4 }}
+    {{- end }}
 spec:
   replicas: {{ .Values.custom.scheduler_replicas }}
   selector:
@@ -112,7 +119,24 @@ spec:
     metadata:
       labels:
         app: volcano-scheduler
+        {{- if .Values.custom.scheduler_podLabels }}
+        {{- toYaml .Values.custom.scheduler_podLabels | nindent 8 }}
+        {{- end }}
     spec:
+      {{- if $scheduler_tolerations }}
+      tolerations: {{- toYaml $scheduler_tolerations | nindent 8 }}
+      {{- end }}
+      {{- if $scheduler_ns }}
+      nodeSelector: {{- toYaml $scheduler_ns | nindent 8 }}
+      {{- end }}
+      {{- if $scheduler_affinity }}
+      affinity:
+        {{- toYaml $scheduler_affinity | nindent 8 }}
+      {{- end }}
+      {{- if $scheduler_sc }}
+      securityContext:
+        {{- toYaml $scheduler_sc | nindent 8 }}
+      {{- end }}
       serviceAccount: {{ .Release.Name }}-scheduler
       priorityClassName: system-cluster-critical
       {{- if .Values.basic.image_pull_secret }}
@@ -122,6 +146,10 @@ spec:
       containers:
         - name: {{ .Release.Name }}-scheduler
           image: {{.Values.basic.scheduler_image_name}}:{{.Values.basic.image_tag_version}}
+          {{- if .Values.custom.scheduler_resources }}
+          resources:
+          {{- toYaml .Values.custom.scheduler_resources | nindent 12 }}
+          {{- end }}
           args:
             - --logtostderr
             - --scheduler-conf=/volcano.scheduler/{{base .Values.basic.scheduler_config_file}}
@@ -141,18 +169,6 @@ spec:
         - name: scheduler-config
           configMap:
             name: {{ .Release.Name }}-scheduler-configmap
-      {{- with .Values.nodeSelector }}
-      nodeSelector:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.affinity }}
-      affinity:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
-      {{- with .Values.tolerations }}
-      tolerations:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
 ---
 apiVersion: v1
 kind: Service

diff --git a/installer/helm/chart/volcano/values.yaml b/installer/helm/chart/volcano/values.yaml
@@ -19,6 +19,77 @@ custom:
   scheduler_replicas: 1
   leader_elect_enable: false
   enabled_admissions: "/jobs/mutate,/jobs/validate,/podgroups/mutate,/pods/validate,/pods/mutate,/queues/mutate,/queues/validate"
-nodeSelector: {}
-affinity: []
-tolerations: []
+
+# Specify affinity for all main Volcano components or per component.
+# For example:
+#
+#  default_affinity:
+#    podAntiAffinity:
+#      preferredDuringSchedulingIgnoredDuringExecution:
+#        - podAffinityTerm:
+#            labelSelector:
+#              matchLabels:
+#                key: value
+#            topologyKey: kubernetes.io/hostname
+#          weight: 49
+  default_affinity: ~
+  admission_affinity: ~
+  controller_affinity: ~
+  scheduler_affinity: ~
+
+# Specify tolerations for all main Volcano components or per component
+# For example:
+#
+#  default_tolerations:
+#  - key: "example-key1"
+#    operator: "Exists"
+#    effect: "NoSchedule"
+  default_tolerations: ~
+  admission_tolerations: ~
+  controller_tolerations: ~
+  scheduler_tolerations: ~
+
+# Specify securityContext for all main Volcano components or per component
+# For example:
+#
+#  default_sc:
+#    runAsUser: 3000
+#    runAsGroup: 3000
+  default_sc: ~
+  scheduler_sc: ~
+  admission_sc: ~
+  controller_sc: ~
+
+# Specify nodeSelector for all main Volcano components or per component
+# For example:
+#
+#  default_ns:
+#    nodetype: criticalservices
+  default_ns: ~
+  admission_ns: ~
+  scheduler_ns: ~
+  controller_ns: ~
+
+
+# Specify labels for Volcano main component deployments and pods
+# For example:
+#
+#  admission_podLabels:
+#    key1: value1
+  admission_podLabels: ~
+  scheduler_podLabels: ~
+  controller_podLabels: ~
+  admission_labels: ~
+  scheduler_labels: ~
+  controller_labels: ~
+
+# Specify resources for Volcano main component deployments and pods
+# For example:
+#
+#  admission_resources:
+#    limits:
+#      cpu: 300m
+#      memory: 300Mi
+  admission_resources: ~
+  scheduler_resources: ~
+  controller_resources: ~