diff --git a/controllers/provider-azure/charts/internal/cloud-provider-config/templates/cloud-provider-config.tpl b/controllers/provider-azure/charts/internal/cloud-provider-config/templates/cloud-provider-config.tpl index d3f08af6e..d26145fba 100644 --- a/controllers/provider-azure/charts/internal/cloud-provider-config/templates/cloud-provider-config.tpl +++ b/controllers/provider-azure/charts/internal/cloud-provider-config/templates/cloud-provider-config.tpl @@ -1,12 +1,13 @@ {{- define "cloud-provider-config"}} cloud: AZUREPUBLICCLOUD -resourceGroup: "{{ .Values.resourceGroup }}" location: "{{ .Values.region }}" -vnetName: "{{ .Values.vnetName }}" -subnetName: "{{ .Values.subnetName }}" -securityGroupName: "{{ .Values.securityGroupName }}" -routeTableName: "{{ .Values.routeTableName }}" primaryAvailabilitySetName: "{{ .Values.availabilitySetName }}" +resourceGroup: "{{ .Values.resourceGroup }}" +routeTableName: "{{ .Values.routeTableName }}" +securityGroupName: "{{ .Values.securityGroupName }}" +loadBalancerSku: "{{ .Values.loadBalancerSku }}" +subnetName: "{{ .Values.subnetName }}" +vnetName: "{{ .Values.vnetName }}" cloudProviderBackoff: true cloudProviderBackoffRetries: 6 cloudProviderBackoffExponent: 1.5 diff --git a/controllers/provider-azure/charts/internal/cloud-provider-config/values.yaml b/controllers/provider-azure/charts/internal/cloud-provider-config/values.yaml index c70ef0c7c..61e29e829 100644 --- a/controllers/provider-azure/charts/internal/cloud-provider-config/values.yaml +++ b/controllers/provider-azure/charts/internal/cloud-provider-config/values.yaml @@ -9,4 +9,5 @@ availabilitySetName: av-set subnetName: sname routeTableName: rtname securityGroupName: sgname +loadBalancerSku: standard region: location \ No newline at end of file diff --git a/controllers/provider-azure/docs/migrate-loadbalancer.md b/controllers/provider-azure/docs/migrate-loadbalancer.md new file mode 100644 index 000000000..84c36c9be --- /dev/null +++ b/controllers/provider-azure/docs/migrate-loadbalancer.md @@ -0,0 +1,89 @@ +# Migrate Azure Shoot Load Balancer from basic to standard SKU + +This guide descibes how to migrate the Load Balancer of an Azure Shoot cluster from the basic SKU to the standard SKU.
+**Be aware:** You need to delete and recreate all services of type Load Balancer, which means that the public ip addresses of your service endpoints will change.
+Please do this only if the Stakeholder really needs to migrate this Shoot to use standard Load Balancers. All new Shoot clusters will automatically use Azure Standard Load Balancers. + +1. Disable temporarily Gardeners reconciliation.
+The Gardener Controller Manager need to be configured to allow ignoring Shoot clusters. +This can be configured in its the `ControllerManagerConfiguration` via the field `.controllers.shoot.respectSyncPeriodOverwrite="true"`. + +```sh +# In the Garden cluster. +kubectl annotate shoot shoot.garden.sapcloud.io/ignore="true" + +# In the Seed cluster. +kubectl -n scale deployment gardener-resource-manager --replicas=0 +``` + +2. Backup all Kubernetes services of type Load Balancer. +```sh +# In the Shoot cluster. +# Determine all Load Balancer services. +kubectl get service --all-namespaces | grep LoadBalancer + +# Backup each Load Balancer service. +echo "---" >> service-backup.yaml && kubectl -n get service -o yaml >> service-backup.yaml +``` + +3. Delete all Load Balancer services. +```sh +# In the Shoot cluster. +kubectl -n delete service +``` + +4. Wait until until Load Balancer is deleted. +Wait until all services of type Load Balancer are deleted and the Azure Load Balancer resource is also deleted. +Check via the Azure Portal if the Load Balancer within the Shoot Resource Group has been deleted. +This should happen automatically after all Kubernetes Load Balancer service are gone within a few minutes. + +Alternatively the Azure cli can be used to check the Load Balancer in the Shoot Resource Group. +The credentials to configure the cli are available on the Seed cluster in the Shoot namespace. +```sh +# In the Seed cluster. +# Fetch the credentials from cloudprovider secret. +kubectl -n get secret cloudprovider -o yaml + +# Configure the Azure cli, with the base64 decoded values of the cloudprovider secret. +az login --service-principal --username --password --tenant +az account set -s + +# Fetch the constantly the Shoot Load Balancer in the Shoot Resource Group. Wait until the resource is gone. +watch 'az network lb show -g shoot---- -n shoot----' + +# Logout. +az logout +``` + +5. Modify the `cloud-povider-config` configmap in the Seed namespace of the Shoot.
+The key `cloudprovider.conf` contains the Kubernetes cloud-provider configuration. +The value is a multiline string. Please change the value of the field `loadBalancerSku` from `basic` to `standard`. +Iff the field does not exists then append `loadBalancerSku: \"standard\"\n` to the value/string. +```sh +# In the Seed cluster. +kubectl -n edit cm cloud-provider-config +``` + +6. Enable Gardeners reconcilation and trigger a reconciliation. +``` +# In the Garden cluster +# Enable reconcilation +kubectl annotate shoot shoot.garden.sapcloud.io/ignore- + +# Trigger reconcilation +kubectl annotate shoot shoot.garden.sapcloud.io/operation="reconcile" +``` +Wait until the cluster has been reconciled. + +6. Recreate the services from the backup file.
+Probably you need to remove some fields from the service defintions e.g. `.spec.clusterIP`, `.metadata.uid` or `.status` etc. +```sh +kubectl apply -f service-backup.yaml +``` + +7. If successful remove backup file. +```sh +# Delete the backup file. +rm -f service-backup.yaml +``` + diff --git a/controllers/provider-azure/pkg/controller/controlplane/valuesprovider.go b/controllers/provider-azure/pkg/controller/controlplane/valuesprovider.go index 99eaa53b3..3e14eb4da 100644 --- a/controllers/provider-azure/pkg/controller/controlplane/valuesprovider.go +++ b/controllers/provider-azure/pkg/controller/controlplane/valuesprovider.go @@ -17,6 +17,7 @@ package controlplane import ( "context" "path/filepath" + "strings" apisazure "github.com/gardener/gardener-extensions/controllers/provider-azure/pkg/apis/azure" azureapihelper "github.com/gardener/gardener-extensions/controllers/provider-azure/pkg/apis/azure/helper" @@ -26,6 +27,7 @@ import ( "github.com/gardener/gardener-extensions/pkg/controller/controlplane" "github.com/gardener/gardener-extensions/pkg/controller/controlplane/genericactuator" "github.com/gardener/gardener-extensions/pkg/util" + kutil "github.com/gardener/gardener/pkg/utils/kubernetes" gardencorev1alpha1 "github.com/gardener/gardener/pkg/apis/core/v1alpha1" extensionsv1alpha1 "github.com/gardener/gardener/pkg/apis/extensions/v1alpha1" @@ -36,6 +38,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apiserver/pkg/authentication/user" @@ -46,6 +49,8 @@ import ( const ( cloudControllerManagerDeploymentName = "cloud-controller-manager" cloudControllerManagerServerName = "cloud-controller-manager-server" + cloudProviderConfigMapName = "cloud-provider-config" + cloudProviderConfigMapKey = "cloudprovider.conf" ) var controlPlaneSecrets = &secrets.Secrets{ @@ -175,8 +180,14 @@ func (vp *valuesProvider) GetConfigChartValues( return nil, errors.Wrapf(err, "could not get service account from secret '%s/%s'", cp.Spec.SecretRef.Namespace, cp.Spec.SecretRef.Name) } + // Determine which kind of LoadBalancer should be configured in the cloud-provider-config. + loadBalancerType, err := determineLoadBalancerType(ctx, vp.client, cp.Namespace) + if err != nil { + return nil, errors.Wrap(err, "could not determine which type of loadbalancer should be used") + } + // Get config chart values - return getConfigChartValues(infraStatus, cp, cluster, auth) + return getConfigChartValues(infraStatus, cp, cluster, auth, loadBalancerType) } // GetControlPlaneChartValues returns the values for the control plane chart applied by the generic actuator. @@ -203,6 +214,7 @@ func getConfigChartValues( cp *extensionsv1alpha1.ControlPlane, cluster *extensionscontroller.Cluster, ca *internal.ClientAuth, + loadBalancerType string, ) (map[string]interface{}, error) { subnetName, availabilitySetName, routeTableName, securityGroupName, err := getInfraNames(infraStatus) if err != nil { @@ -222,6 +234,7 @@ func getConfigChartValues( "availabilitySetName": availabilitySetName, "routeTableName": routeTableName, "securityGroupName": securityGroupName, + "loadBalancerSku": loadBalancerType, "region": cp.Spec.Region, }, nil } @@ -275,3 +288,30 @@ func getInfraNames(infraStatus *apisazure.InfrastructureStatus) (string, string, return nodesSubnet.Name, nodesAvailabilitySet.Name, nodesRouteTable.Name, nodesSecurityGroup.Name, nil } + +func determineLoadBalancerType(ctx context.Context, c client.Client, namespace string) (string, error) { + var ( + cm = &corev1.ConfigMap{} + cmRef = kutil.Key(namespace, cloudProviderConfigMapName) + ) + // Check if a cloud-provider-config configmap already exists. + // If this is not the case it can assume this is a new cluster and use standard LoadBalancers. + if err := c.Get(ctx, cmRef, cm); err != nil { + if apierrors.IsNotFound(err) { + return "standard", nil + } + return "", errors.Wrapf(err, "could not fetch existing %s configmap", cloudProviderConfigMapName) + } + data, ok := cm.Data[cloudProviderConfigMapKey] + if !ok { + return "standard", nil + } + // If the cloud-provider-config does not contain a LoadBalancer type configuration + // then it choose the basic LoadBalancers as they were the former default and + // there is no automatic migration path implemented. + // Anyways it writes the usedLoadBalancer type now explictly to the cloud-privider-config. + if !strings.Contains(data, "loadBalancerSku") || strings.Contains(data, "loadBalancerSku: \"basic\"") { + return "basic", nil + } + return "standard", nil +} diff --git a/controllers/provider-azure/pkg/controller/controlplane/valuesprovider_test.go b/controllers/provider-azure/pkg/controller/controlplane/valuesprovider_test.go index 0ca54cce2..67010f552 100644 --- a/controllers/provider-azure/pkg/controller/controlplane/valuesprovider_test.go +++ b/controllers/provider-azure/pkg/controller/controlplane/valuesprovider_test.go @@ -376,6 +376,16 @@ var _ = Describe("ValuesProvider", func() { "tenantID": []byte(`TenantID`), }, } + cloudProviderConfigKey = client.ObjectKey{Namespace: namespace, Name: cloudProviderConfigMapName} + cloudProviderConfigMap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cloudProviderConfigMapName, + Namespace: namespace, + }, + Data: map[string]string{ + cloudProviderConfigMapKey: "loadBalancerSku: \"standard\"", + }, + } checksums = map[string]string{ gardencorev1alpha1.SecretNameCloudProvider: "8bafb35ff1ac60275d62e1cbd495aceb511fb354f74a20f7d06ecb48b3a68432", @@ -392,6 +402,7 @@ var _ = Describe("ValuesProvider", func() { "resourceGroup": "rg-abcd1234", "vnetName": "vnet-abcd1234", "subnetName": "subnet-abcd1234-nodes", + "loadBalancerSku": "standard", "region": "eu-west-1a", "availabilitySetName": "availability-set-name", "routeTableName": "route-table-name", @@ -430,6 +441,7 @@ var _ = Describe("ValuesProvider", func() { It("should return correct config chart values", func() { // Create mock client client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap)) client.EXPECT().Get(context.TODO(), cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret)) // Create valuesProvider @@ -441,6 +453,7 @@ var _ = Describe("ValuesProvider", func() { // Call GetConfigChartValues method and check the result values, err := vp.GetConfigChartValues(context.TODO(), cp, cluster) + Expect(err).NotTo(HaveOccurred()) Expect(values).To(Equal(configChartValues)) }) @@ -450,6 +463,7 @@ var _ = Describe("ValuesProvider", func() { It("should return error, missing subnet", func() { // Create mock client client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap)) client.EXPECT().Get(context.TODO(), cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret)) // Create valuesProvider @@ -470,6 +484,7 @@ var _ = Describe("ValuesProvider", func() { It("should return error, missing availability set", func() { // Create mock client client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap)) client.EXPECT().Get(context.TODO(), cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret)) // Create valuesProvider @@ -490,6 +505,7 @@ var _ = Describe("ValuesProvider", func() { It("should return error, missing route tables", func() { // Create mock client client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap)) client.EXPECT().Get(context.TODO(), cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret)) // Create valuesProvider @@ -510,6 +526,7 @@ var _ = Describe("ValuesProvider", func() { It("should return error, missing security groups", func() { // Create mock client client := mockclient.NewMockClient(ctrl) + client.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap)) client.EXPECT().Get(context.TODO(), cpSecretKey, &corev1.Secret{}).DoAndReturn(clientGet(cpSecret)) // Create valuesProvider @@ -539,6 +556,76 @@ var _ = Describe("ValuesProvider", func() { Expect(values).To(Equal(ccmChartValues)) }) }) + + Describe("#determineLoadBalancerType", func() { + It("should use standard load balancer, as configured in cloud-provider-config", func() { + c := mockclient.NewMockClient(ctrl) + c.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cloudProviderConfigMap)) + + vc := client.Client(c) + lbType, err := determineLoadBalancerType(context.TODO(), vc, namespace) + Expect(err).NotTo(HaveOccurred()) + Expect(lbType).To(Equal("standard")) + }) + + It("should use standard load balancer, as cloud-provider-config is empty", func() { + var cm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cloudProviderConfigMapName, + Namespace: namespace, + }, + Data: map[string]string{}, + } + + c := mockclient.NewMockClient(ctrl) + c.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm)) + + vc := client.Client(c) + lbType, err := determineLoadBalancerType(context.TODO(), vc, namespace) + Expect(err).NotTo(HaveOccurred()) + Expect(lbType).To(Equal("standard")) + }) + + It("should use basic load balancer, as no lb config in cloud-provider-config", func() { + var cm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cloudProviderConfigMapName, + Namespace: namespace, + }, + Data: map[string]string{ + cloudProviderConfigMapKey: "", + }, + } + + c := mockclient.NewMockClient(ctrl) + c.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm)) + + vc := client.Client(c) + lbType, err := determineLoadBalancerType(context.TODO(), vc, namespace) + Expect(err).NotTo(HaveOccurred()) + Expect(lbType).To(Equal("basic")) + }) + + It("should use basic load balancer, as configured in cloud-provider-config", func() { + var cm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cloudProviderConfigMapName, + Namespace: namespace, + }, + Data: map[string]string{ + cloudProviderConfigMapKey: "loadBalancerSku: \"basic\"", + }, + } + + c := mockclient.NewMockClient(ctrl) + c.EXPECT().Get(context.TODO(), cloudProviderConfigKey, &corev1.ConfigMap{}).DoAndReturn(clientGet(cm)) + + vc := client.Client(c) + lbType, err := determineLoadBalancerType(context.TODO(), vc, namespace) + Expect(err).NotTo(HaveOccurred()) + Expect(lbType).To(Equal("basic")) + }) + }) }) func encode(obj runtime.Object) []byte { @@ -551,6 +638,8 @@ func clientGet(result runtime.Object) interface{} { switch obj.(type) { case *corev1.Secret: *obj.(*corev1.Secret) = *result.(*corev1.Secret) + case *corev1.ConfigMap: + *obj.(*corev1.ConfigMap) = *result.(*corev1.ConfigMap) } return nil }