From fe6e71a0600192705e4c68da7bafd0df6448b17c Mon Sep 17 00:00:00 2001 From: Sven Tasche Date: Wed, 6 Mar 2024 16:12:19 +0100 Subject: [PATCH] feat(base-cluster): upgrade all HRs and enable driftDetection (#773) Tests needed: - [x] upgrade exsting base-cluster HR - [x] install base-cluster on empty kubernetes --- charts/base-cluster/templates/_helmRelease.yaml | 4 +++- .../templates/backup/migrations/velero-4-to-5.yaml | 2 +- charts/base-cluster/templates/backup/velero.yaml | 4 +++- charts/base-cluster/templates/cert-manager/cert-manager.yaml | 4 +++- charts/base-cluster/templates/descheduler/descheduler.yaml | 4 +++- charts/base-cluster/templates/dns/external-dns.yaml | 4 +++- charts/base-cluster/templates/global/reflector.yaml | 4 +++- charts/base-cluster/templates/ingress/nginx.yaml | 4 +++- charts/base-cluster/templates/kyverno/kyverno.yaml | 4 +++- .../policies/kyverno-base-policies/kyverno-policies.yaml | 4 +++- charts/base-cluster/templates/kyverno/validation.tpl | 2 +- .../monitoring/kube-prometheus-stack/oauth-proxy.yaml | 4 +++- .../monitoring/kube-prometheus-stack/prometheus-operator.yaml | 4 +++- charts/base-cluster/templates/monitoring/loki/loki.yaml | 4 +++- .../templates/monitoring/metrics-server/metrics-server.yaml | 4 +++- charts/base-cluster/templates/monitoring/security/trivy.yaml | 4 +++- .../templates/monitoring/tracing/grafana-tempo.yaml | 4 +++- .../templates/monitoring/tracing/opentelemetry-collector.yaml | 4 +++- .../nfs-server-provisioner/nfs-server-provisioner.yaml | 4 +++- 19 files changed, 53 insertions(+), 19 deletions(-) diff --git a/charts/base-cluster/templates/_helmRelease.yaml b/charts/base-cluster/templates/_helmRelease.yaml index 615a8a686..86c110096 100644 --- a/charts/base-cluster/templates/_helmRelease.yaml +++ b/charts/base-cluster/templates/_helmRelease.yaml @@ -1,5 +1,5 @@ {{- define "base-cluster.helm.resourceWithDependencies" -}} -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: {{ .name }} @@ -12,6 +12,8 @@ spec: chart: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "cetic" "chart" "static" "context" .context) | nindent 6 }} interval: 1h + driftDetection: + mode: enabled dependsOn: {{- range $namespace, $name := .dependencies }} - name: {{ $name }} diff --git a/charts/base-cluster/templates/backup/migrations/velero-4-to-5.yaml b/charts/base-cluster/templates/backup/migrations/velero-4-to-5.yaml index f5896cb60..c840bba02 100644 --- a/charts/base-cluster/templates/backup/migrations/velero-4-to-5.yaml +++ b/charts/base-cluster/templates/backup/migrations/velero-4-to-5.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.backup.backupStorageLocations (hasPrefix "4." (dig "spec" "chart" "spec" "version" "" (lookup "helm.toolkit.fluxcd.io/v2beta1" "HelmRelease" "backup" "velero"))) }} +{{- if and .Values.backup.backupStorageLocations (hasPrefix "4." (dig "spec" "chart" "spec" "version" "" (lookup "helm.toolkit.fluxcd.io/v2beta2" "HelmRelease" "backup" "velero"))) }} apiVersion: batch/v1 kind: Job metadata: diff --git a/charts/base-cluster/templates/backup/velero.yaml b/charts/base-cluster/templates/backup/velero.yaml index 19bed3055..c34fe0116 100644 --- a/charts/base-cluster/templates/backup/velero.yaml +++ b/charts/base-cluster/templates/backup/velero.yaml @@ -1,5 +1,5 @@ {{- if .Values.backup.backupStorageLocations }} -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: velero @@ -10,6 +10,8 @@ spec: chart: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "vmware" "chart" "velero" "context" $) | nindent 6 }} interval: 1h + driftDetection: + mode: enabled {{- if .Values.monitoring.prometheus.enabled }} dependsOn: - name: kube-prometheus-stack diff --git a/charts/base-cluster/templates/cert-manager/cert-manager.yaml b/charts/base-cluster/templates/cert-manager/cert-manager.yaml index 212ba6b2e..1212bd83d 100644 --- a/charts/base-cluster/templates/cert-manager/cert-manager.yaml +++ b/charts/base-cluster/templates/cert-manager/cert-manager.yaml @@ -1,4 +1,4 @@ -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: cert-manager @@ -9,6 +9,8 @@ spec: chart: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "jetstack" "chart" "cert-manager" "context" $) | nindent 6 }} interval: 1h + driftDetection: + mode: enabled {{- if .Values.monitoring.prometheus.enabled }} dependsOn: - name: kube-prometheus-stack diff --git a/charts/base-cluster/templates/descheduler/descheduler.yaml b/charts/base-cluster/templates/descheduler/descheduler.yaml index e27e42087..f0500b380 100644 --- a/charts/base-cluster/templates/descheduler/descheduler.yaml +++ b/charts/base-cluster/templates/descheduler/descheduler.yaml @@ -3,7 +3,7 @@ {{- $versionMatrix := dict 18 "0.20.x" 19 "0.21.x" 20 "0.22.x" 21 "0.23.x" 22 "0.24.x" 23 "0.25.x" 24 "0.26.x" 25 "0.27.x" 26 "0.28.x" -}} {{- $latestVersion := .Values.global.helmRepositories.descheduler.charts.descheduler -}} {{- $selectedVersion := (hasKey $versionMatrix $kubeMinorVersion) | ternary (index $versionMatrix $kubeMinorVersion) $latestVersion -}} -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: descheduler @@ -23,6 +23,8 @@ spec: namespace: {{ .Release.Namespace }} version: {{ $selectedVersion }} interval: 1h + driftDetection: + mode: enabled values: cronJobApiVersion: {{ include "common.capabilities.cronjob.apiVersion" . }} startingDeadlineSeconds: 120 diff --git a/charts/base-cluster/templates/dns/external-dns.yaml b/charts/base-cluster/templates/dns/external-dns.yaml index c71626957..0c3776516 100644 --- a/charts/base-cluster/templates/dns/external-dns.yaml +++ b/charts/base-cluster/templates/dns/external-dns.yaml @@ -1,6 +1,6 @@ {{- if .Values.dns.provider -}} {{- $apiVersion := (include "common.capabilities.ingress.apiVersion" .) -}} -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: external-dns @@ -24,6 +24,8 @@ spec: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "bitnami" "chart" "external-dns" "context" $) | nindent 6 }} {{- end }} interval: 1h + driftDetection: + mode: enabled {{- if .Values.monitoring.prometheus.enabled }} dependsOn: - name: kube-prometheus-stack diff --git a/charts/base-cluster/templates/global/reflector.yaml b/charts/base-cluster/templates/global/reflector.yaml index 345330118..fc6fe1d70 100644 --- a/charts/base-cluster/templates/global/reflector.yaml +++ b/charts/base-cluster/templates/global/reflector.yaml @@ -1,5 +1,5 @@ {{- if include "base-cluster.reflector.enabled" (dict "context" .) -}} -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: reflector @@ -10,6 +10,8 @@ spec: chart: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "emberstack" "chart" "reflector" "context" $) | nindent 6 }} interval: 1h + driftDetection: + mode: enabled values: priorityClassName: cluster-components {{- if .Values.global.imageRegistry }} diff --git a/charts/base-cluster/templates/ingress/nginx.yaml b/charts/base-cluster/templates/ingress/nginx.yaml index 4ff5e1494..624782297 100644 --- a/charts/base-cluster/templates/ingress/nginx.yaml +++ b/charts/base-cluster/templates/ingress/nginx.yaml @@ -1,5 +1,5 @@ {{ if .Values.ingress.enabled }} -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: ingress-nginx @@ -10,6 +10,8 @@ spec: chart: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "nginx" "chart" "ingress-nginx" "context" $) | nindent 6 }} interval: 1h + driftDetection: + mode: enabled {{- if .Values.monitoring.prometheus.enabled }} dependsOn: - name: kube-prometheus-stack diff --git a/charts/base-cluster/templates/kyverno/kyverno.yaml b/charts/base-cluster/templates/kyverno/kyverno.yaml index c7eb8bba8..725518a9d 100644 --- a/charts/base-cluster/templates/kyverno/kyverno.yaml +++ b/charts/base-cluster/templates/kyverno/kyverno.yaml @@ -1,5 +1,5 @@ {{- if .Values.kyverno.enabled }} -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: kyverno @@ -10,6 +10,8 @@ spec: chart: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "kyverno" "chart" "kyverno" "context" $) | nindent 6 }} interval: 1h + driftDetection: + mode: enabled {{- if .Values.monitoring.prometheus.enabled }} dependsOn: - name: kube-prometheus-stack diff --git a/charts/base-cluster/templates/kyverno/policies/kyverno-base-policies/kyverno-policies.yaml b/charts/base-cluster/templates/kyverno/policies/kyverno-base-policies/kyverno-policies.yaml index 7df8e36cb..3250047f1 100644 --- a/charts/base-cluster/templates/kyverno/policies/kyverno-base-policies/kyverno-policies.yaml +++ b/charts/base-cluster/templates/kyverno/policies/kyverno-base-policies/kyverno-policies.yaml @@ -1,6 +1,6 @@ {{- if .Values.kyverno.enabled }} # https://github.com/kyverno/kyverno/tree/main/charts/kyverno-policies -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: kyverno-policies @@ -12,6 +12,8 @@ spec: chart: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "kyverno" "chart" "kyverno-policies" "context" $) | nindent 6 }} interval: 1h + driftDetection: + mode: enabled dependsOn: - name: kyverno namespace: kyverno diff --git a/charts/base-cluster/templates/kyverno/validation.tpl b/charts/base-cluster/templates/kyverno/validation.tpl index 4460558d2..9b223f302 100644 --- a/charts/base-cluster/templates/kyverno/validation.tpl +++ b/charts/base-cluster/templates/kyverno/validation.tpl @@ -1,4 +1,4 @@ -{{- $existingKyverno := lookup "helm.toolkit.fluxcd.io/v2beta1" "HelmRelease" "kyverno" "kyverno" -}} +{{- $existingKyverno := lookup "helm.toolkit.fluxcd.io/v2beta2" "HelmRelease" "kyverno" "kyverno" -}} {{- $lastAttemptedRevision := dig "status" "lastAttemptedRevision" "" $existingKyverno }} {{- $lastAppliedRevision := dig "status" "lastAppliedRevision" "" $existingKyverno }} {{- if or $lastAppliedRevision $lastAttemptedRevision -}} diff --git a/charts/base-cluster/templates/monitoring/kube-prometheus-stack/oauth-proxy.yaml b/charts/base-cluster/templates/monitoring/kube-prometheus-stack/oauth-proxy.yaml index 3a3ceb9d8..ef99c03c9 100644 --- a/charts/base-cluster/templates/monitoring/kube-prometheus-stack/oauth-proxy.yaml +++ b/charts/base-cluster/templates/monitoring/kube-prometheus-stack/oauth-proxy.yaml @@ -11,7 +11,7 @@ {{- $port := $backend.port -}} {{- $targetServiceName := printf "%s-%s" (include "common.names.dependency.fullname" (dict "chartName" "kube-prometheus-stack" "chartValues" (dict) "context" (dict "Release" (dict "Name" "kube-prometheus-stack")))) $host -}} {{- $ingress := include "base-cluster.monitoring.ingress.config" (dict "name" $host "context" $) | fromYaml -}} -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: cluster-{{ $host }}-oauth-proxy @@ -23,6 +23,8 @@ spec: chart: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "bitnami" "chart" "oauth2-proxy" "context" $) | nindent 6 }} interval: 1h + driftDetection: + mode: enabled values: redis: enabled: false diff --git a/charts/base-cluster/templates/monitoring/kube-prometheus-stack/prometheus-operator.yaml b/charts/base-cluster/templates/monitoring/kube-prometheus-stack/prometheus-operator.yaml index 5477db26e..949255854 100644 --- a/charts/base-cluster/templates/monitoring/kube-prometheus-stack/prometheus-operator.yaml +++ b/charts/base-cluster/templates/monitoring/kube-prometheus-stack/prometheus-operator.yaml @@ -1,5 +1,5 @@ {{- if .Values.monitoring.prometheus.enabled }} -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: kube-prometheus-stack @@ -11,6 +11,8 @@ spec: chart: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "prometheus" "chart" "kube-prometheus-stack" "context" $) | nindent 6 }} interval: 1h + driftDetection: + mode: enabled install: timeout: 20m0s crds: CreateReplace diff --git a/charts/base-cluster/templates/monitoring/loki/loki.yaml b/charts/base-cluster/templates/monitoring/loki/loki.yaml index b3d2cee9e..605ac057c 100644 --- a/charts/base-cluster/templates/monitoring/loki/loki.yaml +++ b/charts/base-cluster/templates/monitoring/loki/loki.yaml @@ -1,5 +1,5 @@ {{- if and .Values.monitoring.prometheus.enabled .Values.monitoring.loki.enabled -}} -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: loki @@ -11,6 +11,8 @@ spec: chart: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "grafana" "chart" "loki-stack" "context" $) | nindent 6 }} interval: 1h + driftDetection: + mode: enabled install: timeout: 10m0s upgrade: diff --git a/charts/base-cluster/templates/monitoring/metrics-server/metrics-server.yaml b/charts/base-cluster/templates/monitoring/metrics-server/metrics-server.yaml index 52f037c6c..ba3b11c2b 100644 --- a/charts/base-cluster/templates/monitoring/metrics-server/metrics-server.yaml +++ b/charts/base-cluster/templates/monitoring/metrics-server/metrics-server.yaml @@ -1,5 +1,5 @@ {{- if .Values.monitoring.metricsServer.enabled -}} -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: metrics-server @@ -11,6 +11,8 @@ spec: chart: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "bitnami" "chart" "metrics-server" "context" $) | nindent 6 }} interval: 1h + driftDetection: + mode: enabled values: apiService: create: true diff --git a/charts/base-cluster/templates/monitoring/security/trivy.yaml b/charts/base-cluster/templates/monitoring/security/trivy.yaml index b68026233..9ea723ac2 100644 --- a/charts/base-cluster/templates/monitoring/security/trivy.yaml +++ b/charts/base-cluster/templates/monitoring/security/trivy.yaml @@ -1,5 +1,5 @@ {{- if .Values.monitoring.securityScanning.enabled }} -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: trivy @@ -11,6 +11,8 @@ spec: chart: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "trivy" "chart" "trivy-operator" "context" $) | nindent 6 }} interval: 1h + driftDetection: + mode: enabled {{- if .Values.monitoring.prometheus.enabled }} dependsOn: - name: kube-prometheus-stack diff --git a/charts/base-cluster/templates/monitoring/tracing/grafana-tempo.yaml b/charts/base-cluster/templates/monitoring/tracing/grafana-tempo.yaml index 47f0d8f02..345472feb 100644 --- a/charts/base-cluster/templates/monitoring/tracing/grafana-tempo.yaml +++ b/charts/base-cluster/templates/monitoring/tracing/grafana-tempo.yaml @@ -1,5 +1,5 @@ {{- if and .Values.monitoring.tracing.enabled .Values.monitoring.prometheus.enabled -}} -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: grafana-tempo @@ -11,6 +11,8 @@ spec: chart: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "bitnami" "chart" "grafana-tempo" "context" $) | nindent 6 }} interval: 1h + driftDetection: + mode: enabled dependsOn: - name: kube-prometheus-stack namespace: monitoring diff --git a/charts/base-cluster/templates/monitoring/tracing/opentelemetry-collector.yaml b/charts/base-cluster/templates/monitoring/tracing/opentelemetry-collector.yaml index 93852e810..9635c18cd 100644 --- a/charts/base-cluster/templates/monitoring/tracing/opentelemetry-collector.yaml +++ b/charts/base-cluster/templates/monitoring/tracing/opentelemetry-collector.yaml @@ -1,5 +1,5 @@ {{- if and .Values.monitoring.tracing.enabled .Values.monitoring.prometheus.enabled -}} -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: open-telemetry-collector @@ -11,6 +11,8 @@ spec: chart: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "open-telemetry" "chart" "opentelemetry-collector" "context" $) | nindent 6 }} interval: 1h + driftDetection: + mode: enabled dependsOn: - name: kube-prometheus-stack namespace: monitoring diff --git a/charts/base-cluster/templates/nfs-server-provisioner/nfs-server-provisioner.yaml b/charts/base-cluster/templates/nfs-server-provisioner/nfs-server-provisioner.yaml index 638c40181..145c78385 100644 --- a/charts/base-cluster/templates/nfs-server-provisioner/nfs-server-provisioner.yaml +++ b/charts/base-cluster/templates/nfs-server-provisioner/nfs-server-provisioner.yaml @@ -1,5 +1,5 @@ {{- if .Values.storage.readWriteMany.enabled }} -apiVersion: helm.toolkit.fluxcd.io/v2beta1 +apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: name: nfs-server-provisioner @@ -11,6 +11,8 @@ spec: chart: spec: {{- include "base-cluster.helm.chartSpec" (dict "repo" "nfs-server-provisioner" "chart" "nfs-server-provisioner" "context" $) | nindent 6 }} interval: 1h + driftDetection: + mode: enabled values: storageClass: name: {{ .Values.storage.readWriteMany.storageClass.name }}