Nat Gateway Cluster Egress Options (#266)

* Added ManagedNat Egress type Signed-off-by: Gordonby <gordon.byers@microsoft.com> * UI and Playwright test for NatGw * cspell typos Signed-off-by: Gordonby <gordon.byers@microsoft.com> * preview features Signed-off-by: Gordonby <gordon.byers@microsoft.com> * added preview and docs message * byo vnet param config * disabling byo for managedNatGw * bicep decorators * add NAT gateway for starter cluster json config * userassignedgw * fixing playwright test * private endpoint custom nic name
Azure · May 16, 2022 · 4a79cd1 · 4a79cd1
1 parent f1d6d26
commit 4a79cd1
Show file tree

Hide file tree

Showing 12 changed files with 232 additions and 23 deletions.
diff --git a/.github/workflows_dep/AksDeploy-Basic.parameters.json b/.github/workflows_dep/AksDeploy-Basic.parameters.json
@@ -37,6 +37,9 @@
         },
         "registries_sku": {
             "value": "Standard"
+        },
+        "aksOutboundTrafficType" : {
+            "value": "managedNATGateway"
         }
     }
 }
diff --git a/.github/workflows_dep/AksDeploy-ByoVnet.parameters.json b/.github/workflows_dep/AksDeploy-ByoVnet.parameters.json
@@ -95,6 +95,12 @@
         },
         "dnsZoneId" : {
             "value" : "/this/parameter/is/overridden/in/my/script/valuefromsecrets"
+        },
+        "aksOutboundTrafficType" : {
+            "value": "userAssignedNATGateway"
+        },
+        "createNatGateway" : {
+            "value": false
         }
     }
 }
diff --git a/.github/workflows_dep/AksDeploy-Private.parameters.json b/.github/workflows_dep/AksDeploy-Private.parameters.json
@@ -88,6 +88,12 @@
           },
           "CreateNetworkSecurityGroupFlowLogs": {
             "value": true
+          },
+          "aksOutboundTrafficType" : {
+              "value": "userAssignedNATGateway"
+          },
+          "createNatGateway" : {
+              "value": true
           }
     }
 }
diff --git a/.github/workflows_dep/regressionparams/managedNatGw.json b/.github/workflows_dep/regressionparams/managedNatGw.json
@@ -0,0 +1,21 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "resourceName": {
+      "value": "az-k8s-natgw"
+    },
+    "agentVMSize": {
+      "value": "Standard_DS3_v2"
+    },
+    "aksOutboundTrafficType" : {
+      "value": "managedNATGateway"
+    },
+    "natGwIpCount" : {
+      "value": 3
+    },
+    "natGwIdleTimeout" : {
+      "value": 5
+    }
+  }
+}
diff --git a/bicep/main.bicep b/bicep/main.bicep
@@ -150,6 +150,9 @@ module network './network.bicep' = if (custom_vnet) {
     networkSecurityGroups: CreateNetworkSecurityGroups
     CreateNsgFlowLogs: CreateNetworkSecurityGroups && CreateNetworkSecurityGroupFlowLogs
     ingressApplicationGatewayPublic: empty(privateIpApplicationGateway)
+    natGateway: createNatGateway
+    natGatewayIdleTimeoutMins: natGwIdleTimeout
+    natGatewayPublicIps: natGwIpCount
   }
 }
 output CustomVnetId string = custom_vnet ? network.outputs.vnetId : ''
@@ -917,6 +920,27 @@ param AutoscaleProfile object = {
   'skip-nodes-with-system-pods': 'true'
 }
 
+@allowed([
+  'loadBalancer'
+  'managedNATGateway'
+  'userAssignedNATGateway'
+])
+@description('Outbound traffic type for the egress traffic of your cluster')
+param aksOutboundTrafficType string = 'loadBalancer'
+
+@description('Create a new Nat Gateway, applies to custom networking only')
+param createNatGateway bool = false
+
+@minValue(1)
+@maxValue(16)
+@description('The effective outbound IP resources of the cluster NAT gateway')
+param natGwIpCount int = 2
+
+@minValue(4)
+@maxValue(120)
+@description('Outbound flow idle timeout in minutes for NatGw')
+param natGwIdleTimeout int = 30
+
 @description('System Pool presets are derived from the recommended system pool specs')
 var systemPoolPresets = {
   'Cost-Optimised' : {
@@ -1082,6 +1106,13 @@ var aksProperties = {
     serviceCidr: serviceCidr
     dnsServiceIP: dnsServiceIP
     dockerBridgeCidr: dockerBridgeCidr
+    outboundType: aksOutboundTrafficType
+    natGatewayProfile: aksOutboundTrafficType == 'managedNATGateway' ? {
+      managedOutboundIPProfile: {
+        count: natGwIpCount
+      }
+      idleTimeoutInMinutes: natGwIdleTimeout
+    } : {}
   }
   disableLocalAccounts: AksDisableLocalAccounts && enable_aad
   autoUpgradeProfile: !empty(upgradeChannel) ? {
@@ -1101,7 +1132,7 @@ var azureDefenderSecurityProfile = {
   }
 }
 
-resource aks 'Microsoft.ContainerService/managedClusters@2021-10-01' = {
+resource aks 'Microsoft.ContainerService/managedClusters@2022-03-02-preview' = {
   name: 'aks-${resourceName}'
   location: location
   properties: DefenderForContainers && omsagent ? union(aksProperties,azureDefenderSecurityProfile) : aksProperties

diff --git a/bicep/network.bicep b/bicep/network.bicep
@@ -30,6 +30,11 @@ param privateLinkAkvId string = ''
 param acrPrivatePool bool = false
 param acrAgentPoolSubnetAddressPrefix string = ''
 
+//NatGatewayEgress
+param natGateway bool = false
+param natGatewayPublicIps int = 2
+param natGatewayIdleTimeoutMins int
+
 //Bastion
 param bastion bool =false
 param bastionSubnetAddressPrefix string = ''
@@ -119,6 +124,10 @@ var aks_baseSubnet =  {
     }, privateLinks ? {
       privateEndpointNetworkPolicies: 'Disabled'
       privateLinkServiceNetworkPolicies: 'Enabled'
+    } : {}, natGateway ? {
+      natGateway: {
+        id: natGw.id
+      }
     } : {}, azureFirewalls ? {
       routeTable: {
         id: vnet_udr.id //resourceId('Microsoft.Network/routeTables', routeFwTableName)
@@ -132,8 +141,6 @@ var subnets_2 = privateLinks ? concat(array(subnets_1), array(private_link_subne
 var subnets_3 = acrPrivatePool ? concat(array(subnets_2), array(acrpool_subnet)) : array(subnets_2)
 var subnets_4 = bastion ? concat(array(subnets_3), array(bastion_subnet)) : array(subnets_3)
 
-// DONT create appgw subnet, the addon will create it for us
-
 var final_subnets = ingressApplicationGateway ? concat(array(subnets_4), array(appgw_subnet)) : array(subnets_4)
 
 var vnetName = 'vnet-${resourceName}'
@@ -171,11 +178,11 @@ resource aks_vnet_cont 'Microsoft.Network/virtualNetworks/subnets/providers/role
 
 /*   --------------------------------------------------------------------------  Private Link for ACR      */
 var privateLinkAcrName = 'pl-acr-${resourceName}'
-resource privateLinkAcr 'Microsoft.Network/privateEndpoints@2021-03-01' = if (!empty(privateLinkAcrId)) {
+resource privateLinkAcr 'Microsoft.Network/privateEndpoints@2021-08-01' = if (!empty(privateLinkAcrId)) {
   name: privateLinkAcrName
   location: location
   properties: {
-    //customNetworkInterfaceName: 'nic-${privateLinkAcrName}' needs AllowPrivateEndpointCustomNicName registered in subscription in order to rename
+    customNetworkInterfaceName: 'nic-${privateLinkAcrName}'
     privateLinkServiceConnections: [
       {
         name: 'Acr-Connection'
@@ -211,7 +218,7 @@ resource privateDnsAcrLink 'Microsoft.Network/privateDnsZones/virtualNetworkLink
   }
 }
 
-resource privateDnsAcrZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2021-03-01' = if (!empty(privateLinkAcrId))  {
+resource privateDnsAcrZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2021-08-01' = if (!empty(privateLinkAcrId))  {
   parent: privateLinkAcr
   name: 'default'
   properties: {
@@ -229,11 +236,11 @@ resource privateDnsAcrZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZo
 
 /*   --------------------------------------------------------------------------  Private Link for KeyVault      */
 var privateLinkAkvName = 'pl-akv-${resourceName}'
-resource privateLinkAkv 'Microsoft.Network/privateEndpoints@2021-03-01' = if (!empty(privateLinkAkvId)) {
+resource privateLinkAkv 'Microsoft.Network/privateEndpoints@2021-08-01' = if (!empty(privateLinkAkvId)) {
   name: privateLinkAkvName
   location: location
   properties: {
-    //customNetworkInterfaceName: 'nic-${privateLinkAkvName}' needs AllowPrivateEndpointCustomNicName registered in subscription in order to rename
+    customNetworkInterfaceName: 'nic-${privateLinkAkvName}'
     privateLinkServiceConnections: [
       {
         name: 'Akv-Connection'
@@ -269,7 +276,7 @@ resource privateDnsAkvLink 'Microsoft.Network/privateDnsZones/virtualNetworkLink
   }
 }
 
-resource privateDnsAkvZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2021-03-01' = if (!empty(privateLinkAkvId))  {
+resource privateDnsAkvZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2021-08-01' = if (!empty(privateLinkAkvId))  {
   parent: privateLinkAkv
   name: 'default'
   properties: {
@@ -437,5 +444,34 @@ module nsgPrivateLinks 'nsg.bicep' = if(privateLinks && networkSecurityGroups) {
   ]
 }
 
+resource natGwIp 'Microsoft.Network/publicIPAddresses@2021-08-01' =  [for i in range(0, natGatewayPublicIps): if(natGateway) {
+  name: 'pip-${natGwName}-${i+1}'
+  location: location
+  sku: {
+    name: 'Standard'
+  }
+  zones: !empty(availabilityZones) ? availabilityZones : []
+  properties: {
+    publicIPAllocationMethod: 'Static'
+  }
+}]
 
+var natGwName = 'ng-${resourceName}'
+resource natGw 'Microsoft.Network/natGateways@2021-08-01' = if(natGateway) {
+  name: natGwName
+  location: location
+  sku: {
+    name: 'Standard'
+  }
+  zones: !empty(availabilityZones) ? availabilityZones : []
+  properties: {
+    publicIpAddresses: [for i in range(0, natGatewayPublicIps): {
+      id: natGwIp[i].id
+    }]
+    idleTimeoutInMinutes: natGatewayIdleTimeoutMins
+  }
+  dependsOn: [
+    natGwIp
+  ]
+}
 
diff --git a/helper/.playwrighttests/helper-test-managednatgw.spec.js b/helper/.playwrighttests/helper-test-managednatgw.spec.js
@@ -0,0 +1,30 @@
+const { test, expect } = require('@playwright/test');
+const { matchers } = require('playwright-expect');
+
+// add custom matchers
+expect.extend(matchers);
+
+test('managed-natgw-option-is-not-the-default', async ({ page }) => {
+
+  await page.goto('http://localhost:3000/AKS-Construction');
+
+  // Click the 4th Tab in the portal Navigation Pivot (network)
+  await page.click('[data-testid="portalnav-Pivot"] > button:nth-child(4)');
+
+  //Check default value
+  const dropdown = await page.waitForSelector('[data-testid="net-aksEgressType"]')
+  await expect(dropdown).toBeVisible()
+  await expect(dropdown).toMatchText('Load Balancer')
+
+  // Click the 1st Tab in the portal Navigation Pivot (network)
+  await page.click('[data-testid="portalnav-Pivot"] > button:nth-child(1)');
+
+  // //Check parameter is absent
+  await page.waitForSelector('[data-testid="deploy-deploycmd"]')
+  const clitextbox = await page.$('[data-testid="deploy-deploycmd"]')
+  await expect(clitextbox).toBeVisible()
+  await expect(clitextbox).not.toContainText('aksOutboundTrafficType')
+
+});
+
+  //TODO: Change value and check (this is a real pain with the DropDown control)
diff --git a/helper/src/components/addonsTab.js b/helper/src/components/addonsTab.js
@@ -31,8 +31,6 @@ export default function ({ tabValues, updateFn, featureFlag, invalidArray }) {
 
             </Stack.Item>
 
-
-
             <Stack.Item align="center" styles={{ root: { width: '700px' }}}>
                 <Checkbox disabled={addons.registry === "none" || !net.vnetprivateend} checked={addons.acrPrivatePool} onChange={(ev, v) => updateFn("acrPrivatePool", v)} label={<Text>Create ACR Private Agent Pool (private link only) (preview limited regions <a target="_new" href="https://docs.microsoft.com/azure/container-registry/tasks-agent-pools">docs</a>)</Text>} />
                 <Stack horizontal styles={{ root: { marginLeft: "50px" } }}>

diff --git a/helper/src/components/deployTab.js b/helper/src/components/deployTab.js
@@ -78,6 +78,20 @@ export default function DeployTab({ defaults, updateFn, tabValues, invalidArray,
 
   const preview_params = {
     ...(addons.gitops !== "none" && { gitops: addons.gitops }),
+    ...(net.vnet_opt === "default" && net.aksOutboundTrafficType === 'managedNATGateway' && {
+      ...(net.aksOutboundTrafficType !== defaults.net.aksOutboundTrafficType && {aksOutboundTrafficType: net.aksOutboundTrafficType}),
+      ...(net.natGwIpCount !== defaults.net.natGwIpCount && {natGwIpCount: net.natGwIpCount}),
+      ...(net.natGwIdleTimeout !== defaults.net.natGwIdleTimeout && {natGwIdleTimeout: net.natGwIdleTimeout})
+    }),
+    ...(net.vnet_opt === "custom" && net.aksOutboundTrafficType === 'userAssignedNATGateway' && {
+      ...({createNatGateway: true}),
+      ...(net.aksOutboundTrafficType !== defaults.net.aksOutboundTrafficType && {aksOutboundTrafficType: net.aksOutboundTrafficType}),
+      ...(net.natGwIpCount !== defaults.net.natGwIpCount && {natGwIpCount: net.natGwIpCount}),
+      ...(net.natGwIdleTimeout !== defaults.net.natGwIdleTimeout && {natGwIdleTimeout: net.natGwIdleTimeout})
+    }),
+    ...(net.vnet_opt === "byo" && {
+      ...(net.aksOutboundTrafficType !== defaults.net.aksOutboundTrafficType && {aksOutboundTrafficType: net.aksOutboundTrafficType})
+    }),
     ...(net.vnet_opt === "custom" && net.vnetprivateend && {
       ...(addons.registry !== "none" && {
         ...(addons.acrPrivatePool !== defaults.addons.acrPrivatePool && {acrPrivatePool: addons.acrPrivatePool}),

diff --git a/helper/src/components/networkTab.js b/helper/src/components/networkTab.js
@@ -1,6 +1,6 @@
 
 import React, { useState } from 'react';
-import { Image, ImageFit, Link, Separator, TextField, DirectionalHint, Callout, Stack, Text, Label, ChoiceGroup, Checkbox, MessageBar, MessageBarType } from '@fluentui/react';
+import { Image, ImageFit, Link, Separator, TextField, DirectionalHint, Callout, Stack, Text, Label, ChoiceGroup, Checkbox, MessageBar, MessageBarType, Dropdown, Slider } from '@fluentui/react';
 import { adv_stackstyle, hasError, getError } from './common'
 
 const columnProps = {
@@ -38,26 +38,83 @@ export default function NetworkTab ({ tabValues, updateFn, invalidArray, feature
             <Separator className="notopmargin" />
 
             <Stack.Item>
-                <Label>Deploy Azure firewall for your cluster egress (Custom VNet Only)</Label>
-                {hasError(invalidArray, 'afw') &&
-                    <MessageBar messageBarType={MessageBarType.error}>{getError(invalidArray, 'afw')}</MessageBar>
-                }
-                <Checkbox styles={{ root: { marginLeft: '50px', marginTop: '10 !important' } }} disabled={false} errorMessage={getError(invalidArray, 'afw')} checked={net.afw} onChange={(ev, v) => updateFn("afw", v)} label="Implement Azure Firewall & UDR next hop" />
-
+                <Label>Uses a private IP address from your VNet to access your dependent Azure service, such as Azure KeyVault, Azure Container Registry etc</Label>
+                <Checkbox styles={{ root: { marginLeft: '50px', marginTop: '0 !important' } }} disabled={false} checked={net.vnetprivateend} onChange={(ev, v) => updateFn("vnetprivateend", v)} label="Enable Private Link" />
             </Stack.Item>
 
             <Separator className="notopmargin" />
 
             <Stack.Item>
-                <Label>Uses a private IP address from your VNet to access your dependent Azure service, such as Azure KeyVault, Azure Container Registry etc</Label>
-                <Checkbox styles={{ root: { marginLeft: '50px', marginTop: '0 !important' } }} disabled={false} checked={net.vnetprivateend} onChange={(ev, v) => updateFn("vnetprivateend", v)} label="Enable Private Link" />
+                <Label>Use Azure Bastion to facilitate RDP/SSH public internet inbound access into your virtual network</Label>
+                <Checkbox inputProps={{ "data-testid": "network-bastion-Checkbox"}} styles={{ root: { marginLeft: '50px', marginTop: '0 !important' } }} disabled={false} checked={net.bastion} onChange={(ev, v) => updateFn("bastion", v)} label="Enable Azure Bastion" />
+            </Stack.Item>
+
+            <Separator className="notopmargin" />
+
+            <Stack.Item >
+                <Label>AKS Traffic Egress</Label>
+
+                <Stack horizontal tokens={{ childrenGap: 50 }}>
+                    <Stack.Item>
+                        <MessageBar messageBarType={MessageBarType.warning}>Nat Gateway for AKS egress is currently a preview feature <a target="_target" href="https://docs.microsoft.com/azure/aks/nat-gateway">docs</a></MessageBar>
+                        {hasError(invalidArray, 'aksOutboundTrafficType') &&
+                            <MessageBar messageBarType={MessageBarType.error}>{getError(invalidArray, 'aksOutboundTrafficType')}</MessageBar>
+                        }
+                        <ChoiceGroup
+                            styles={{ root: { marginLeft: '50px' } }}
+                            selectedKey={net.aksOutboundTrafficType}
+                            errorMessage={getError(invalidArray, 'aksOutboundTrafficType')}
+                            data-testid="net-aksEgressType"
+                            options={[
+                                { key: 'loadBalancer', text: 'Load Balancer'  },
+                                { key: 'managedNATGateway', text: 'Managed NAT Gateway' },
+                                { key: 'userAssignedNATGateway', text: 'Assigned NAT Gateway'}
+                            ]}
+                            onChange={(ev, { key }) => updateFn("aksOutboundTrafficType", key)}
+                        />
+                    </Stack.Item>
+                    <Stack.Item>
+                        <Checkbox //simple "read-only" checkbox that derives its values from other settings
+                            styles={{ root: { marginBottom: '10px' }}}
+                            checked={net.vnet_opt === 'custom' && net.aksOutboundTrafficType === 'userAssignedNATGateway'}
+                            disabled={true}
+                            label="Create NAT Gateway for AKS Subnet (Custom VNet Only)"
+                        />
+                        <Slider
+                            disabled={net.aksOutboundTrafficType==='loadBalancer' || net.vnet_opt === 'byo'}
+                            buttonProps={{ "data-testid": "net-natGwIp-slider"}}
+                            styles={{ root: { width: 450 } }}
+                            label={'Nat Gateway Ip Count'} min={1}  max={16} step={1}
+                            value={net.natGwIpCount} showValue={true}
+                            onChange={(val, range) => updateFn("natGwIpCount", val)}
+                        />
+
+                        <Slider
+                            disabled={net.aksOutboundTrafficType==='loadBalancer' || net.vnet_opt === 'byo'}
+                            buttonProps={{ "data-testid": "net-natGwTimeout-slider"}}
+                            styles={{ root: { width: 450 } }}
+                            label={'Nat Gateway Idle Timeout (Minutes)'} min={5}  max={120} step={1}
+                            value={net.natGwIdleTimeout} showValue={true}
+                            onChange={(val, range) => updateFn("natGwIdleTimeout", val)}
+                        />
+                    </Stack.Item>
+                </Stack>
             </Stack.Item>
 
             <Separator className="notopmargin" />
 
             <Stack.Item>
-                <Label>Use Azure Bastion to facilitate RDP/SSH public internet inbound access into your virtual network</Label>
-                <Checkbox inputProps={{ "data-testid": "network-bastion-Checkbox"}} styles={{ root: { marginLeft: '50px', marginTop: '0 !important' } }} disabled={false} checked={net.bastion} onChange={(ev, v) => updateFn("bastion", v)} label="Enable Azure Bastion" />
+                <Label>Deploy Azure firewall for your cluster egress (Custom VNet Only)</Label>
+                {hasError(invalidArray, 'afw') &&
+                    <MessageBar messageBarType={MessageBarType.error}>{getError(invalidArray, 'afw')}</MessageBar>
+                }
+                <Checkbox
+                    styles={{ root: { marginLeft: '50px', marginTop: '10 !important' } }}
+                    disabled={net.vnet_opt !== 'custom'}
+                    errorMessage={getError(invalidArray, 'afw')}
+                    checked={net.afw}
+                    onChange={(ev, v) => updateFn("afw", v)}
+                    label="Implement Azure Firewall & UDR next hop" />
             </Stack.Item>
 
             <Separator className="notopmargin" />