Azure: Move to standard loadbalancer

controllers/provider-azure/charts/internal/cloud-provider-config/templates/cloud-provider-config.tpl

@@ -1,12 +1,13 @@
 {{- define "cloud-provider-config"}}
 cloud: AZUREPUBLICCLOUD
-resourceGroup: "{{ .Values.resourceGroup }}"
 location: "{{ .Values.region }}"
-vnetName: "{{ .Values.vnetName }}"
-subnetName: "{{ .Values.subnetName }}"
-securityGroupName: "{{ .Values.securityGroupName }}"
-routeTableName: "{{ .Values.routeTableName }}"
 primaryAvailabilitySetName: "{{ .Values.availabilitySetName }}"
+resourceGroup: "{{ .Values.resourceGroup }}"
+routeTableName: "{{ .Values.routeTableName }}"
+securityGroupName: "{{ .Values.securityGroupName }}"
+loadBalancerSku: "{{ .Values.loadBalancerSku }}"
+subnetName: "{{ .Values.subnetName }}"
+vnetName: "{{ .Values.vnetName }}"
 cloudProviderBackoff: true
 cloudProviderBackoffRetries: 6
 cloudProviderBackoffExponent: 1.5

controllers/provider-azure/charts/internal/cloud-provider-config/values.yaml

@@ -9,4 +9,5 @@ availabilitySetName: av-set
 subnetName: sname
 routeTableName: rtname
 securityGroupName: sgname
+loadBalancerSku: standard
 region: location

controllers/provider-azure/docs/migrate-loadbalancer.md

@@ -0,0 +1,89 @@
+# Migrate Azure Shoot Load Balancer from basic to standard SKU
+
+This guide descibes how to migrate the Load Balancer of an Azure Shoot cluster from the basic SKU to the standard SKU.<br/>
+**Be aware:** You need to delete and recreate all services of type Load Balancer, which means that the public ip addresses of your service endpoints will change.<br/>
+Please do this only if the Stakeholder really needs to migrate this Shoot to use standard Load Balancers. All new Shoot clusters will automatically use Azure Standard Load Balancers.
+
+1. Disable temporarily Gardeners reconciliation.<br>
+The Gardener Controller Manager need to be configured to allow ignoring Shoot clusters.
+This can be configured in its the `ControllerManagerConfiguration` via the field `.controllers.shoot.respectSyncPeriodOverwrite="true"`.
+
+```sh
+# In the Garden cluster.
+kubectl annotate shoot <shoot-name> shoot.garden.sapcloud.io/ignore="true"
+
+# In the Seed cluster.
+kubectl -n <shoot-namespace> scale deployment gardener-resource-manager --replicas=0
+```
+
+2. Backup all Kubernetes services of type Load Balancer.
+```sh
+# In the Shoot cluster.
+# Determine all Load Balancer services.
+kubectl get service --all-namespaces | grep LoadBalancer
+
+# Backup each Load Balancer service.
+echo "---" >> service-backup.yaml && kubectl -n <namespace> get service <service-name> -o yaml >> service-backup.yaml
+```
+
+3. Delete all Load Balancer services.
+```sh
+# In the Shoot cluster.
+kubectl -n <namespace> delete service <service-name>
+```
+
+4. Wait until until Load Balancer is deleted.
+Wait until all services of type Load Balancer are deleted and the Azure Load Balancer resource is also deleted.
+Check via the Azure Portal if the Load Balancer within the Shoot Resource Group has been deleted.
+This should happen automatically after all Kubernetes Load Balancer service are gone within a few minutes.
+
+Alternatively the Azure cli can be used to check the Load Balancer in the Shoot Resource Group.
+The credentials to configure the cli are available on the Seed cluster in the Shoot namespace.
+```sh
+# In the Seed cluster.
+# Fetch the credentials from cloudprovider secret.
+kubectl -n <shoot-namespace> get secret cloudprovider -o yaml
+
+# Configure the Azure cli, with the base64 decoded values of the cloudprovider secret.
+az login --service-principal --username <clientID> --password <clientSecret> --tenant <tenantID>
+az account set -s <subscriptionID>
+
+# Fetch the constantly the Shoot Load Balancer in the Shoot Resource Group. Wait until the resource is gone.
+watch 'az network lb show -g shoot--<project-name>--<shoot-name> -n shoot--<project-name>--<shoot-name>'
+
+# Logout.
+az logout
+```
+
+5. Modify the `cloud-povider-config` configmap in the Seed namespace of the Shoot.<br/>
+The key `cloudprovider.conf` contains the Kubernetes cloud-provider configuration.
+The value is a multiline string. Please change the value of the field `loadBalancerSku` from `basic` to `standard`.
+Iff the field does not exists then append `loadBalancerSku: \"standard\"\n` to the value/string.
+```sh
+# In the Seed cluster.
+kubectl -n <shoot-namespace> edit cm cloud-provider-config
+```
+
+6. Enable Gardeners reconcilation and trigger a reconciliation.
+```
+# In the Garden cluster
+# Enable reconcilation
+kubectl annotate shoot <shoot-name> shoot.garden.sapcloud.io/ignore-
+
+# Trigger reconcilation
+kubectl annotate shoot <shoot-name> shoot.garden.sapcloud.io/operation="reconcile"
+```
+Wait until the cluster has been reconciled.
+
+6. Recreate the services from the backup file.<br/>
+Probably you need to remove some fields from the service defintions e.g. `.spec.clusterIP`, `.metadata.uid` or `.status` etc.
+```sh
+kubectl apply -f service-backup.yaml
+```
+
+7. If successful remove backup file.
+```sh
+# Delete the backup file.
+rm -f service-backup.yaml
+```
+

controllers/provider-azure/pkg/controller/controlplane/valuesprovider.go

@@ -17,6 +17,7 @@ package controlplane
 import (
 	"context"
 	"path/filepath"
+	"strings"
 
 	apisazure "github.com/gardener/gardener-extensions/controllers/provider-azure/pkg/apis/azure"
 	azureapihelper "github.com/gardener/gardener-extensions/controllers/provider-azure/pkg/apis/azure/helper"
@@ -26,6 +27,7 @@ import (
 	"github.com/gardener/gardener-extensions/pkg/controller/controlplane"
 	"github.com/gardener/gardener-extensions/pkg/controller/controlplane/genericactuator"
 	"github.com/gardener/gardener-extensions/pkg/util"
+	kutil "github.com/gardener/gardener/pkg/utils/kubernetes"
 
 	gardencorev1alpha1 "github.com/gardener/gardener/pkg/apis/core/v1alpha1"
 	extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
@@ -36,6 +38,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apiserver/pkg/authentication/user"
@@ -46,6 +49,8 @@ import (
 const (
 	cloudControllerManagerDeploymentName = "cloud-controller-manager"
 	cloudControllerManagerServerName     = "cloud-controller-manager-server"
+	cloudProviderConfigMapName           = "cloud-provider-config"
+	cloudProviderConfigMapKey            = "cloudprovider.conf"
 )
 
 var controlPlaneSecrets = &secrets.Secrets{
@@ -174,8 +179,14 @@ func (vp *valuesProvider) GetConfigChartValues(
 		return nil, errors.Wrapf(err, "could not get service account from secret '%s/%s'", cp.Spec.SecretRef.Namespace, cp.Spec.SecretRef.Name)
 	}
 
+	// Determine which kind of LoadBalancer should be configured in the cloud-provider-config.
+	loadBalancerType, err := determineLoadBalancerType(ctx, vp.client, cp.Namespace)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not determine which type of loadbalancer should be used")
+	}
+
 	// Get config chart values
-	return getConfigChartValues(infraStatus, cp, cluster, auth)
+	return getConfigChartValues(infraStatus, cp, cluster, auth, loadBalancerType)
 }
 
 // GetControlPlaneChartValues returns the values for the control plane chart applied by the generic actuator.
@@ -202,6 +213,7 @@ func getConfigChartValues(
 	cp *extensionsv1alpha1.ControlPlane,
 	cluster *extensionscontroller.Cluster,
 	ca *internal.ClientAuth,
+	loadBalancerType string,
 ) (map[string]interface{}, error) {
 	subnetName, availabilitySetName, routeTableName, securityGroupName, err := getInfraNames(infraStatus)
 	if err != nil {
@@ -221,6 +233,7 @@ func getConfigChartValues(
 		"availabilitySetName": availabilitySetName,
 		"routeTableName":      routeTableName,
 		"securityGroupName":   securityGroupName,
+		"loadBalancerSku":     loadBalancerType,
 		"region":              cp.Spec.Region,
 	}, nil
 }
@@ -274,3 +287,30 @@ func getInfraNames(infraStatus *apisazure.InfrastructureStatus) (string, string,
 
 	return nodesSubnet.Name, nodesAvailabilitySet.Name, nodesRouteTable.Name, nodesSecurityGroup.Name, nil
 }
+
+func determineLoadBalancerType(ctx context.Context, c client.Client, namespace string) (string, error) {
+	var (
+		cm    = &corev1.ConfigMap{}
+		cmRef = kutil.Key(namespace, cloudProviderConfigMapName)
+	)
+	// Check if a cloud-provider-config configmap already exists.
+	// If this is not the case it can assume this is a new cluster and use standard LoadBalancers.
+	if err := c.Get(ctx, cmRef, cm); err != nil {
+		if apierrors.IsNotFound(err) {
+			return "standard", nil
+		}
+		return "", errors.Wrapf(err, "could not fetch existing %s configmap", cloudProviderConfigMapName)
+	}
+	data, ok := cm.Data[cloudProviderConfigMapKey]
+	if !ok {
+		return "standard", nil
+	}
+	// If the cloud-provider-config does not contain a LoadBalancer type configuration
+	// then it choose the basic LoadBalancers as they were the former default and
+	// there is no automatic migration path implemented.
+	// Anyways it writes the usedLoadBalancer type now explictly to the cloud-privider-config.
+	if !strings.Contains(data, "loadBalancerSku") || strings.Contains(data, "loadBalancerSku: \"basic\"") {
+		return "basic", nil
+	}
+	return "standard", nil
+}

controllers/provider-azure/pkg/controller/controlplane/valuesprovider_test.go

@@ -376,6 +376,16 @@ var _ = Describe("ValuesProvider", func() {
 				"tenantID":       []byte(`TenantID`),
 			},
 		}
+		cloudProviderConfigKey = client.ObjectKey{Namespace: namespace, Name: cloudProviderConfigMapName}
+		cloudProviderConfigMap = &corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      cloudProviderConfigMapName,
+				Namespace: namespace,
+			},
+			Data: map[string]string{
+				cloudProviderConfigMapKey: "loadBalancerSku: \"standard\"",
+			},
+		}
 
 		checksums = map[string]string{
 			gardencorev1alpha1.SecretNameCloudProvider: "8bafb35ff1ac60275d62e1cbd495aceb511fb354f74a20f7d06ecb48b3a68432",
@@ -392,6 +402,7 @@ var _ = Describe("ValuesProvider", func() {
 			"resourceGroup":       "rg-abcd1234",
 			"vnetName":            "vnet-abcd1234",
 			"subnetName":          "subnet-abcd1234-nodes",
+			"loadBalancerSku":     "standard",
 			"region":              "eu-west-1a",
 			"availabilitySetName": "availability-set-name",
 			"routeTableName":      "route-table-name",
@@ -430,6 +441,7 @@ var _ = Describe("ValuesProvider", func() {
 		It("should return correct config chart values", func() {
 			// Create mock client
 			client := mockclient.NewMockClient(ctrl)
+			client.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap))
 			client.EXPECT().Get(context.TODO(), cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret))
 
 			// Create valuesProvider
@@ -441,6 +453,7 @@ var _ = Describe("ValuesProvider", func() {
 
 			// Call GetConfigChartValues method and check the result
 			values, err := vp.GetConfigChartValues(context.TODO(), cp, cluster)
+
 			Expect(err).NotTo(HaveOccurred())
 			Expect(values).To(Equal(configChartValues))
 		})
@@ -450,6 +463,7 @@ var _ = Describe("ValuesProvider", func() {
 		It("should return error, missing subnet", func() {
 			// Create mock client
 			client := mockclient.NewMockClient(ctrl)
+			client.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap))
 			client.EXPECT().Get(context.TODO(), cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret))
 
 			// Create valuesProvider
@@ -470,6 +484,7 @@ var _ = Describe("ValuesProvider", func() {
 		It("should return error, missing availability set", func() {
 			// Create mock client
 			client := mockclient.NewMockClient(ctrl)
+			client.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap))
 			client.EXPECT().Get(context.TODO(), cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret))
 
 			// Create valuesProvider
@@ -490,6 +505,7 @@ var _ = Describe("ValuesProvider", func() {
 		It("should return error, missing route tables", func() {
 			// Create mock client
 			client := mockclient.NewMockClient(ctrl)
+			client.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap))
 			client.EXPECT().Get(context.TODO(), cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret))
 
 			// Create valuesProvider
@@ -510,6 +526,7 @@ var _ = Describe("ValuesProvider", func() {
 		It("should return error, missing security groups", func() {
 			// Create mock client
 			client := mockclient.NewMockClient(ctrl)
+			client.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap))
 			client.EXPECT().Get(context.TODO(), cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret))
 
 			// Create valuesProvider
@@ -539,6 +556,76 @@ var _ = Describe("ValuesProvider", func() {
 			Expect(values).To(Equal(ccmChartValues))
 		})
 	})
+
+	Describe("#determineLoadBalancerType", func() {
+		It("should use standard load balancer, as configured in cloud-provider-config", func() {
+			c := mockclient.NewMockClient(ctrl)
+			c.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap))
+
+			vc := client.Client(c)
+			lbType, err := determineLoadBalancerType(context.TODO(), vc, namespace)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(lbType).To(Equal("standard"))
+		})
+
+		It("should use standard load balancer, as cloud-provider-config is empty", func() {
+			var cm = &corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      cloudProviderConfigMapName,
+					Namespace: namespace,
+				},
+				Data: map[string]string{},
+			}
+
+			c := mockclient.NewMockClient(ctrl)
+			c.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm))
+
+			vc := client.Client(c)
+			lbType, err := determineLoadBalancerType(context.TODO(), vc, namespace)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(lbType).To(Equal("standard"))
+		})
+
+		It("should use basic load balancer, as no lb config in cloud-provider-config", func() {
+			var cm = &corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      cloudProviderConfigMapName,
+					Namespace: namespace,
+				},
+				Data: map[string]string{
+					cloudProviderConfigMapKey: "",
+				},
+			}
+
+			c := mockclient.NewMockClient(ctrl)
+			c.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm))
+
+			vc := client.Client(c)
+			lbType, err := determineLoadBalancerType(context.TODO(), vc, namespace)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(lbType).To(Equal("basic"))
+		})
+
+		It("should use basic load balancer, as configured in cloud-provider-config", func() {
+			var cm = &corev1.ConfigMap{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      cloudProviderConfigMapName,
+					Namespace: namespace,
+				},
+				Data: map[string]string{
+					cloudProviderConfigMapKey: "loadBalancerSku: \"basic\"",
+				},
+			}
+
+			c := mockclient.NewMockClient(ctrl)
+			c.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm))
+
+			vc := client.Client(c)
+			lbType, err := determineLoadBalancerType(context.TODO(), vc, namespace)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(lbType).To(Equal("basic"))
+		})
+	})
 })
 
 func encode(obj runtime.Object) []byte {
@@ -551,6 +638,8 @@ func clientGet(result runtime.Object) interface{} {
 		switch obj.(type) {
 		case *corev1.Secret:
 			*obj.(*corev1.Secret) = *result.(*corev1.Secret)
+		case *corev1.ConfigMap:
+			*obj.(*corev1.ConfigMap) = *result.(*corev1.ConfigMap)
 		}
 		return nil
 	}