diff --git a/controllers/provider-azure/charts/internal/cloud-provider-config/templates/cloud-provider-config.tpl b/controllers/provider-azure/charts/internal/cloud-provider-config/templates/cloud-provider-config.tpl
index d3f08af6e..d26145fba 100644
--- a/controllers/provider-azure/charts/internal/cloud-provider-config/templates/cloud-provider-config.tpl
+++ b/controllers/provider-azure/charts/internal/cloud-provider-config/templates/cloud-provider-config.tpl
@@ -1,12 +1,13 @@
{{- define "cloud-provider-config"}}
cloud: AZUREPUBLICCLOUD
-resourceGroup: "{{ .Values.resourceGroup }}"
location: "{{ .Values.region }}"
-vnetName: "{{ .Values.vnetName }}"
-subnetName: "{{ .Values.subnetName }}"
-securityGroupName: "{{ .Values.securityGroupName }}"
-routeTableName: "{{ .Values.routeTableName }}"
primaryAvailabilitySetName: "{{ .Values.availabilitySetName }}"
+resourceGroup: "{{ .Values.resourceGroup }}"
+routeTableName: "{{ .Values.routeTableName }}"
+securityGroupName: "{{ .Values.securityGroupName }}"
+loadBalancerSku: "{{ .Values.loadBalancerSku }}"
+subnetName: "{{ .Values.subnetName }}"
+vnetName: "{{ .Values.vnetName }}"
cloudProviderBackoff: true
cloudProviderBackoffRetries: 6
cloudProviderBackoffExponent: 1.5
diff --git a/controllers/provider-azure/charts/internal/cloud-provider-config/values.yaml b/controllers/provider-azure/charts/internal/cloud-provider-config/values.yaml
index c70ef0c7c..61e29e829 100644
--- a/controllers/provider-azure/charts/internal/cloud-provider-config/values.yaml
+++ b/controllers/provider-azure/charts/internal/cloud-provider-config/values.yaml
@@ -9,4 +9,5 @@ availabilitySetName: av-set
subnetName: sname
routeTableName: rtname
securityGroupName: sgname
+loadBalancerSku: standard
region: location
\ No newline at end of file
diff --git a/controllers/provider-azure/docs/migrate-loadbalancer.md b/controllers/provider-azure/docs/migrate-loadbalancer.md
new file mode 100644
index 000000000..84c36c9be
--- /dev/null
+++ b/controllers/provider-azure/docs/migrate-loadbalancer.md
@@ -0,0 +1,89 @@
+# Migrate Azure Shoot Load Balancer from basic to standard SKU
+
+This guide descibes how to migrate the Load Balancer of an Azure Shoot cluster from the basic SKU to the standard SKU.
+**Be aware:** You need to delete and recreate all services of type Load Balancer, which means that the public ip addresses of your service endpoints will change.
+Please do this only if the Stakeholder really needs to migrate this Shoot to use standard Load Balancers. All new Shoot clusters will automatically use Azure Standard Load Balancers.
+
+1. Disable temporarily Gardeners reconciliation.
+The Gardener Controller Manager need to be configured to allow ignoring Shoot clusters.
+This can be configured in its the `ControllerManagerConfiguration` via the field `.controllers.shoot.respectSyncPeriodOverwrite="true"`.
+
+```sh
+# In the Garden cluster.
+kubectl annotate shoot shoot.garden.sapcloud.io/ignore="true"
+
+# In the Seed cluster.
+kubectl -n scale deployment gardener-resource-manager --replicas=0
+```
+
+2. Backup all Kubernetes services of type Load Balancer.
+```sh
+# In the Shoot cluster.
+# Determine all Load Balancer services.
+kubectl get service --all-namespaces | grep LoadBalancer
+
+# Backup each Load Balancer service.
+echo "---" >> service-backup.yaml && kubectl -n get service -o yaml >> service-backup.yaml
+```
+
+3. Delete all Load Balancer services.
+```sh
+# In the Shoot cluster.
+kubectl -n delete service
+```
+
+4. Wait until until Load Balancer is deleted.
+Wait until all services of type Load Balancer are deleted and the Azure Load Balancer resource is also deleted.
+Check via the Azure Portal if the Load Balancer within the Shoot Resource Group has been deleted.
+This should happen automatically after all Kubernetes Load Balancer service are gone within a few minutes.
+
+Alternatively the Azure cli can be used to check the Load Balancer in the Shoot Resource Group.
+The credentials to configure the cli are available on the Seed cluster in the Shoot namespace.
+```sh
+# In the Seed cluster.
+# Fetch the credentials from cloudprovider secret.
+kubectl -n get secret cloudprovider -o yaml
+
+# Configure the Azure cli, with the base64 decoded values of the cloudprovider secret.
+az login --service-principal --username --password --tenant
+az account set -s
+
+# Fetch the constantly the Shoot Load Balancer in the Shoot Resource Group. Wait until the resource is gone.
+watch 'az network lb show -g shoot---- -n shoot----'
+
+# Logout.
+az logout
+```
+
+5. Modify the `cloud-povider-config` configmap in the Seed namespace of the Shoot.
+The key `cloudprovider.conf` contains the Kubernetes cloud-provider configuration.
+The value is a multiline string. Please change the value of the field `loadBalancerSku` from `basic` to `standard`.
+Iff the field does not exists then append `loadBalancerSku: \"standard\"\n` to the value/string.
+```sh
+# In the Seed cluster.
+kubectl -n edit cm cloud-provider-config
+```
+
+6. Enable Gardeners reconcilation and trigger a reconciliation.
+```
+# In the Garden cluster
+# Enable reconcilation
+kubectl annotate shoot shoot.garden.sapcloud.io/ignore-
+
+# Trigger reconcilation
+kubectl annotate shoot shoot.garden.sapcloud.io/operation="reconcile"
+```
+Wait until the cluster has been reconciled.
+
+6. Recreate the services from the backup file.
+Probably you need to remove some fields from the service defintions e.g. `.spec.clusterIP`, `.metadata.uid` or `.status` etc.
+```sh
+kubectl apply -f service-backup.yaml
+```
+
+7. If successful remove backup file.
+```sh
+# Delete the backup file.
+rm -f service-backup.yaml
+```
+
diff --git a/controllers/provider-azure/pkg/controller/controlplane/valuesprovider.go b/controllers/provider-azure/pkg/controller/controlplane/valuesprovider.go
index 99eaa53b3..3e14eb4da 100644
--- a/controllers/provider-azure/pkg/controller/controlplane/valuesprovider.go
+++ b/controllers/provider-azure/pkg/controller/controlplane/valuesprovider.go
@@ -17,6 +17,7 @@ package controlplane
import (
"context"
"path/filepath"
+ "strings"
apisazure "github.com/gardener/gardener-extensions/controllers/provider-azure/pkg/apis/azure"
azureapihelper "github.com/gardener/gardener-extensions/controllers/provider-azure/pkg/apis/azure/helper"
@@ -26,6 +27,7 @@ import (
"github.com/gardener/gardener-extensions/pkg/controller/controlplane"
"github.com/gardener/gardener-extensions/pkg/controller/controlplane/genericactuator"
"github.com/gardener/gardener-extensions/pkg/util"
+ kutil "github.com/gardener/gardener/pkg/utils/kubernetes"
gardencorev1alpha1 "github.com/gardener/gardener/pkg/apis/core/v1alpha1"
extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1"
@@ -36,6 +38,7 @@ import (
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apiserver/pkg/authentication/user"
@@ -46,6 +49,8 @@ import (
const (
cloudControllerManagerDeploymentName = "cloud-controller-manager"
cloudControllerManagerServerName = "cloud-controller-manager-server"
+ cloudProviderConfigMapName = "cloud-provider-config"
+ cloudProviderConfigMapKey = "cloudprovider.conf"
)
var controlPlaneSecrets = &secrets.Secrets{
@@ -175,8 +180,14 @@ func (vp *valuesProvider) GetConfigChartValues(
return nil, errors.Wrapf(err, "could not get service account from secret '%s/%s'", cp.Spec.SecretRef.Namespace, cp.Spec.SecretRef.Name)
}
+ // Determine which kind of LoadBalancer should be configured in the cloud-provider-config.
+ loadBalancerType, err := determineLoadBalancerType(ctx, vp.client, cp.Namespace)
+ if err != nil {
+ return nil, errors.Wrap(err, "could not determine which type of loadbalancer should be used")
+ }
+
// Get config chart values
- return getConfigChartValues(infraStatus, cp, cluster, auth)
+ return getConfigChartValues(infraStatus, cp, cluster, auth, loadBalancerType)
}
// GetControlPlaneChartValues returns the values for the control plane chart applied by the generic actuator.
@@ -203,6 +214,7 @@ func getConfigChartValues(
cp *extensionsv1alpha1.ControlPlane,
cluster *extensionscontroller.Cluster,
ca *internal.ClientAuth,
+ loadBalancerType string,
) (map[string]interface{}, error) {
subnetName, availabilitySetName, routeTableName, securityGroupName, err := getInfraNames(infraStatus)
if err != nil {
@@ -222,6 +234,7 @@ func getConfigChartValues(
"availabilitySetName": availabilitySetName,
"routeTableName": routeTableName,
"securityGroupName": securityGroupName,
+ "loadBalancerSku": loadBalancerType,
"region": cp.Spec.Region,
}, nil
}
@@ -275,3 +288,30 @@ func getInfraNames(infraStatus *apisazure.InfrastructureStatus) (string, string,
return nodesSubnet.Name, nodesAvailabilitySet.Name, nodesRouteTable.Name, nodesSecurityGroup.Name, nil
}
+
+func determineLoadBalancerType(ctx context.Context, c client.Client, namespace string) (string, error) {
+ var (
+ cm = &corev1.ConfigMap{}
+ cmRef = kutil.Key(namespace, cloudProviderConfigMapName)
+ )
+ // Check if a cloud-provider-config configmap already exists.
+ // If this is not the case it can assume this is a new cluster and use standard LoadBalancers.
+ if err := c.Get(ctx, cmRef, cm); err != nil {
+ if apierrors.IsNotFound(err) {
+ return "standard", nil
+ }
+ return "", errors.Wrapf(err, "could not fetch existing %s configmap", cloudProviderConfigMapName)
+ }
+ data, ok := cm.Data[cloudProviderConfigMapKey]
+ if !ok {
+ return "standard", nil
+ }
+ // If the cloud-provider-config does not contain a LoadBalancer type configuration
+ // then it choose the basic LoadBalancers as they were the former default and
+ // there is no automatic migration path implemented.
+ // Anyways it writes the usedLoadBalancer type now explictly to the cloud-privider-config.
+ if !strings.Contains(data, "loadBalancerSku") || strings.Contains(data, "loadBalancerSku: \"basic\"") {
+ return "basic", nil
+ }
+ return "standard", nil
+}
diff --git a/controllers/provider-azure/pkg/controller/controlplane/valuesprovider_test.go b/controllers/provider-azure/pkg/controller/controlplane/valuesprovider_test.go
index 0ca54cce2..67010f552 100644
--- a/controllers/provider-azure/pkg/controller/controlplane/valuesprovider_test.go
+++ b/controllers/provider-azure/pkg/controller/controlplane/valuesprovider_test.go
@@ -376,6 +376,16 @@ var _ = Describe("ValuesProvider", func() {
"tenantID": []byte(`TenantID`),
},
}
+ cloudProviderConfigKey = client.ObjectKey{Namespace: namespace, Name: cloudProviderConfigMapName}
+ cloudProviderConfigMap = &corev1.ConfigMap{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: cloudProviderConfigMapName,
+ Namespace: namespace,
+ },
+ Data: map[string]string{
+ cloudProviderConfigMapKey: "loadBalancerSku: \"standard\"",
+ },
+ }
checksums = map[string]string{
gardencorev1alpha1.SecretNameCloudProvider: "8bafb35ff1ac60275d62e1cbd495aceb511fb354f74a20f7d06ecb48b3a68432",
@@ -392,6 +402,7 @@ var _ = Describe("ValuesProvider", func() {
"resourceGroup": "rg-abcd1234",
"vnetName": "vnet-abcd1234",
"subnetName": "subnet-abcd1234-nodes",
+ "loadBalancerSku": "standard",
"region": "eu-west-1a",
"availabilitySetName": "availability-set-name",
"routeTableName": "route-table-name",
@@ -430,6 +441,7 @@ var _ = Describe("ValuesProvider", func() {
It("should return correct config chart values", func() {
// Create mock client
client := mockclient.NewMockClient(ctrl)
+ client.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap))
client.EXPECT().Get(context.TODO(), cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret))
// Create valuesProvider
@@ -441,6 +453,7 @@ var _ = Describe("ValuesProvider", func() {
// Call GetConfigChartValues method and check the result
values, err := vp.GetConfigChartValues(context.TODO(), cp, cluster)
+
Expect(err).NotTo(HaveOccurred())
Expect(values).To(Equal(configChartValues))
})
@@ -450,6 +463,7 @@ var _ = Describe("ValuesProvider", func() {
It("should return error, missing subnet", func() {
// Create mock client
client := mockclient.NewMockClient(ctrl)
+ client.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap))
client.EXPECT().Get(context.TODO(), cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret))
// Create valuesProvider
@@ -470,6 +484,7 @@ var _ = Describe("ValuesProvider", func() {
It("should return error, missing availability set", func() {
// Create mock client
client := mockclient.NewMockClient(ctrl)
+ client.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap))
client.EXPECT().Get(context.TODO(), cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret))
// Create valuesProvider
@@ -490,6 +505,7 @@ var _ = Describe("ValuesProvider", func() {
It("should return error, missing route tables", func() {
// Create mock client
client := mockclient.NewMockClient(ctrl)
+ client.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap))
client.EXPECT().Get(context.TODO(), cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret))
// Create valuesProvider
@@ -510,6 +526,7 @@ var _ = Describe("ValuesProvider", func() {
It("should return error, missing security groups", func() {
// Create mock client
client := mockclient.NewMockClient(ctrl)
+ client.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap))
client.EXPECT().Get(context.TODO(), cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret))
// Create valuesProvider
@@ -539,6 +556,76 @@ var _ = Describe("ValuesProvider", func() {
Expect(values).To(Equal(ccmChartValues))
})
})
+
+ Describe("#determineLoadBalancerType", func() {
+ It("should use standard load balancer, as configured in cloud-provider-config", func() {
+ c := mockclient.NewMockClient(ctrl)
+ c.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap))
+
+ vc := client.Client(c)
+ lbType, err := determineLoadBalancerType(context.TODO(), vc, namespace)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(lbType).To(Equal("standard"))
+ })
+
+ It("should use standard load balancer, as cloud-provider-config is empty", func() {
+ var cm = &corev1.ConfigMap{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: cloudProviderConfigMapName,
+ Namespace: namespace,
+ },
+ Data: map[string]string{},
+ }
+
+ c := mockclient.NewMockClient(ctrl)
+ c.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm))
+
+ vc := client.Client(c)
+ lbType, err := determineLoadBalancerType(context.TODO(), vc, namespace)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(lbType).To(Equal("standard"))
+ })
+
+ It("should use basic load balancer, as no lb config in cloud-provider-config", func() {
+ var cm = &corev1.ConfigMap{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: cloudProviderConfigMapName,
+ Namespace: namespace,
+ },
+ Data: map[string]string{
+ cloudProviderConfigMapKey: "",
+ },
+ }
+
+ c := mockclient.NewMockClient(ctrl)
+ c.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm))
+
+ vc := client.Client(c)
+ lbType, err := determineLoadBalancerType(context.TODO(), vc, namespace)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(lbType).To(Equal("basic"))
+ })
+
+ It("should use basic load balancer, as configured in cloud-provider-config", func() {
+ var cm = &corev1.ConfigMap{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: cloudProviderConfigMapName,
+ Namespace: namespace,
+ },
+ Data: map[string]string{
+ cloudProviderConfigMapKey: "loadBalancerSku: \"basic\"",
+ },
+ }
+
+ c := mockclient.NewMockClient(ctrl)
+ c.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm))
+
+ vc := client.Client(c)
+ lbType, err := determineLoadBalancerType(context.TODO(), vc, namespace)
+ Expect(err).NotTo(HaveOccurred())
+ Expect(lbType).To(Equal("basic"))
+ })
+ })
})
func encode(obj runtime.Object) []byte {
@@ -551,6 +638,8 @@ func clientGet(result runtime.Object) interface{} {
switch obj.(type) {
case *corev1.Secret:
*obj.(*corev1.Secret) = *result.(*corev1.Secret)
+ case *corev1.ConfigMap:
+ *obj.(*corev1.ConfigMap) = *result.(*corev1.ConfigMap)
}
return nil
}