feat(eks): connect all custom resources to the cluster VPC (#10200)

Allow all our lambda handlers to be provisioned inside the cluster VPC.

The `KubectlProvider` handlers were already placed inside the VPC is they could have, the missing was to include the `ClusterHandler`. This is now possible via the `placeClusterHandlerInVpc` (names are welcome) property.

Default value remains `false` because if the VPC happens to be isolated (i.e no outbound internet access) this would break the deployment. (See #12171)

Closes #9509 

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

packages/@aws-cdk/aws-eks/README.md

@@ -433,6 +433,8 @@ new eks.Cluster(this, 'HelloEKS', {
 });
 ```
 
+> Note: Isolated VPCs (i.e with no internet access) are not currently supported. See https://github.com/aws/aws-cdk/issues/12171
+
 If you do not specify a VPC, one will be created on your behalf, which you can then access via `cluster.vpc`. The cluster VPC will be associated to any EKS managed capacity (i.e Managed Node Groups and Fargate Profiles).
 
 If you allocate self managed capacity, you can specify which subnets should the auto-scaling group use:
@@ -444,8 +446,7 @@ cluster.addAutoScalingGroupCapacity('nodes', {
 });
 ```
 
-In addition to the cluster and the capacity, there are two additional components you might want to
-provision within a VPC.
+There are two additional components you might want to provision within the VPC.
 
 #### Kubectl Handler
 
@@ -459,7 +460,18 @@ If the endpoint does not expose private access (via `EndpointAccess.PUBLIC`) **o
 
 #### Cluster Handler
 
-The `ClusterHandler` is a Lambda function responsible to interact the EKS API in order to control the cluster lifecycle. At the moment, this function cannot be provisioned inside the VPC. See [Attach all Lambda Function to a VPC](https://github.com/aws/aws-cdk/issues/9509) for more details.
+The `ClusterHandler` is a Lambda function responsible to interact with the EKS API in order to control the cluster lifecycle. To provision this function inside the VPC, set the `placeClusterHandlerInVpc` property to `true`. This will place the function inside the private subnets of the VPC based on the selection strategy specified in the [`vpcSubnets`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-eks.Cluster.html#vpcsubnetsspan-classapi-icon-api-icon-experimental-titlethis-api-element-is-experimental-it-may-change-without-noticespan) property.
+
+You can configure the environment of this function by specifying it at cluster instantiation. For example, this can be useful in order to configure an http proxy:
+
+```ts
+const cluster = new eks.Cluster(this, 'hello-eks', {
+  version: eks.KubernetesVersion.V1_18,
+  clusterHandlerEnvironment: {
+    'http_proxy': 'http://proxy.myproxy.com'
+  }
+});
+```
 
 ### Kubectl Support
 
@@ -1122,6 +1134,5 @@ Kubernetes [endpoint access](#endpoint-access), you must also specify:
 ## Known Issues and Limitations
 
 * [One cluster per stack](https://github.com/aws/aws-cdk/issues/10073)
-* [Object pruning](https://github.com/aws/aws-cdk/issues/10495)
 * [Service Account dependencies](https://github.com/aws/aws-cdk/issues/9910)
-* [Attach all Lambda Functions to VPC](https://github.com/aws/aws-cdk/issues/9509)
+* [Support isolated VPCs](https://github.com/aws/aws-cdk/issues/12171)

packages/@aws-cdk/aws-eks/lib/cluster-resource-provider.ts

@@ -1,4 +1,5 @@
 import * as path from 'path';
+import * as ec2 from '@aws-cdk/aws-ec2';
 import * as iam from '@aws-cdk/aws-iam';
 import * as lambda from '@aws-cdk/aws-lambda';
 import { Duration, NestedStack, Stack } from '@aws-cdk/core';
@@ -17,6 +18,21 @@ export interface ClusterResourceProviderProps {
    * The IAM role to assume in order to interact with the cluster.
    */
   readonly adminRole: iam.IRole;
+
+  /**
+   * The VPC to provision the functions in.
+   */
+  readonly vpc?: ec2.IVpc;
+
+  /**
+   * The subnets to place the functions in.
+   */
+  readonly subnets?: ec2.ISubnet[];
+
+  /**
+   * Environment to add to the handler.
+   */
+  readonly environment?: { [key: string]: string };
 }
 
 /**
@@ -46,8 +62,11 @@ export class ClusterResourceProvider extends NestedStack {
       code: lambda.Code.fromAsset(HANDLER_DIR),
       description: 'onEvent handler for EKS cluster resource provider',
       runtime: HANDLER_RUNTIME,
+      environment: props.environment,
       handler: 'index.onEvent',
       timeout: Duration.minutes(1),
+      vpc: props.subnets ? props.vpc : undefined,
+      vpcSubnets: props.subnets ? { subnets: props.subnets } : undefined,
     });
 
     const isComplete = new lambda.Function(this, 'IsCompleteHandler', {
@@ -56,13 +75,17 @@ export class ClusterResourceProvider extends NestedStack {
       runtime: HANDLER_RUNTIME,
       handler: 'index.isComplete',
       timeout: Duration.minutes(1),
+      vpc: props.subnets ? props.vpc : undefined,
+      vpcSubnets: props.subnets ? { subnets: props.subnets } : undefined,
     });
 
     this.provider = new cr.Provider(this, 'Provider', {
       onEventHandler: onEvent,
       isCompleteHandler: isComplete,
       totalTimeout: Duration.hours(1),
       queryInterval: Duration.minutes(1),
+      vpc: props.subnets ? props.vpc : undefined,
+      vpcSubnets: props.subnets ? { subnets: props.subnets } : undefined,
     });
 
     props.adminRole.grant(onEvent.role!, 'sts:AssumeRole');

packages/@aws-cdk/aws-eks/lib/cluster-resource.ts

@@ -21,6 +21,8 @@ export interface ClusterResourceProps {
   readonly endpointPublicAccess: boolean;
   readonly publicAccessCidrs?: string[];
   readonly vpc: ec2.IVpc;
+  readonly environment?: { [key: string]: string };
+  readonly subnets?: ec2.ISubnet[];
   readonly secretsEncryptionKey?: kms.IKey;
 }
 
@@ -57,6 +59,9 @@ export class ClusterResource extends CoreConstruct {
 
     const provider = ClusterResourceProvider.getOrCreate(this, {
       adminRole: this.adminRole,
+      subnets: props.subnets,
+      vpc: props.vpc,
+      environment: props.environment,
     });
 
     const resource = new CustomResource(this, 'Resource', {

packages/@aws-cdk/aws-eks/lib/cluster.ts

@@ -428,6 +428,13 @@ export interface ClusterOptions extends CommonClusterOptions {
    */
   readonly kubectlEnvironment?: { [key: string]: string };
 
+  /**
+   * Custom environment variables when interacting with the EKS endpoint to manage the cluster lifecycle.
+   *
+   * @default - No environment variables.
+   */
+  readonly clusterHandlerEnvironment?: { [key: string]: string };
+
   /**
    * An AWS Lambda Layer which includes `kubectl`, Helm and the AWS CLI.
    *
@@ -468,6 +475,14 @@ export interface ClusterOptions extends CommonClusterOptions {
    * @default true
    */
   readonly prune?: boolean;
+
+  /**
+   * If set to true, the cluster handler functions will be placed in the private subnets
+   * of the cluster vpc, subject to the `vpcSubnets` selection strategy.
+   *
+   * @default false
+   */
+  readonly placeClusterHandlerInVpc?: boolean;
 }
 
 /**
@@ -859,7 +874,6 @@ export class Cluster extends ClusterBase {
 
   /**
    * Custom environment variables when running `kubectl` against this cluster.
-   * @default - no additional environment variables
    */
   public readonly kubectlEnvironment?: { [key: string]: string };
 
@@ -1020,8 +1034,15 @@ export class Cluster extends ClusterBase {
       throw new Error('Vpc must contain private subnets when public endpoint access is restricted');
     }
 
+    const placeClusterHandlerInVpc = props.placeClusterHandlerInVpc ?? false;
+
+    if (placeClusterHandlerInVpc && privateSubents.length === 0) {
+      throw new Error('Cannot place cluster handler in the VPC since no private subnets could be selected');
+    }
+
     const resource = this._clusterResource = new ClusterResource(this, 'Resource', {
       name: this.physicalName,
+      environment: props.clusterHandlerEnvironment,
       roleArn: this.role.roleArn,
       version: props.version.version,
       resourcesVpcConfig: {
@@ -1041,6 +1062,7 @@ export class Cluster extends ClusterBase {
       publicAccessCidrs: this.endpointAccess._config.publicCidrs,
       secretsEncryptionKey: props.secretsEncryptionKey,
       vpc: this.vpc,
+      subnets: placeClusterHandlerInVpc ? privateSubents : undefined,
     });
 
     if (this.endpointAccess._config.privateAccess && privateSubents.length !== 0) {

packages/@aws-cdk/aws-eks/lib/kubectl-provider.ts

@@ -97,6 +97,8 @@ export class KubectlProvider extends NestedStack {
 
     const provider = new cr.Provider(this, 'Provider', {
       onEventHandler: handler,
+      vpc: cluster.kubectlPrivateSubnets ? cluster.vpc : undefined,
+      vpcSubnets: cluster.kubectlPrivateSubnets ? { subnets: cluster.kubectlPrivateSubnets } : undefined,
     });
 
     this.serviceToken = provider.serviceToken;