Fix Shared ADF Lambda Layer builds and add move to ARM-64 Lambdas (#680)

**Why?**

The Lambda Layer that ADF builds included the `bootstrap/adf-build/shared`
folder as its source.

Unfortunately, this resulted in 388 MB of unzipped dependencies.
While the maximum limit for AWS Lambda Layers is set at 250 MB.

**What?**

After going through each of the lambda functions that use this Lambda Layer,
I realized that they only require the files located in
`bootstrap/adf-build/shared/python`.

Since this implied no longer requiring AWS CDK in the Lambda Layer, I also
used this opportunity to move to the ARM-64 Lambdas in ADF.

The new Lambda Layer is 26 MB.

To build ARM-64 on CodeBuild, I've added the Amazon managed container that
enables cross-architecture builds on Docker. That will launch and
configure Qemu correctly, so ARM-64 Docker images can execute correctly.
I've also fixed the build metadata and updated the `sam build` process to
use a container instead. As without it, it could not resolve the dependencies
required for PyYaml and others specifically for ARM-64.

Lastly, I've updated the shared lambda layer permissions, so we have
control in the stack who can access and build on top of it.

linters/custom-adf-dict.txt

@@ -4,6 +4,7 @@ adf
 adfconfig
 awscli
 backoff
+binfmt
 bitnami
 boto
 boto3

src/lambda_codebase/account_bootstrap.py

@@ -9,20 +9,22 @@
 """
 
 import os
-import boto3
 
+import boto3
 from botocore.exceptions import ClientError
-from logger import configure_logger
+
+# ADF imports
 from errors import (
     AccountCreationNotFinishedError,
     GenericAccountConfigureError,
     ParameterNotFoundError,
 )
-from parameter_store import ParameterStore
 from cloudformation import CloudFormation
+from logger import configure_logger
+from parameter_store import ParameterStore
+from partition import get_partition
 from s3 import S3
 from sts import STS
-from partition import get_partition
 
 # Globals taken from the lambda environment variables
 S3_BUCKET = os.environ["S3_BUCKET_NAME"]

src/lambda_codebase/account_processing/configure_account_alias.py

@@ -6,9 +6,12 @@
 """
 
 import os
-from sts import STS
+
 from aws_xray_sdk.core import patch_all
+
+# ADF imports
 from logger import configure_logger
+from sts import STS
 
 patch_all()
 

src/lambda_codebase/account_processing/configure_account_ou.py

@@ -4,10 +4,12 @@
 """
 Moves an account to the specified OU.
 """
-from organizations import Organizations
 import boto3
 from aws_xray_sdk.core import patch_all
+
+# ADF imports
 from logger import configure_logger
+from organizations import Organizations
 
 
 patch_all()

src/lambda_codebase/account_processing/configure_account_regions.py

@@ -6,9 +6,10 @@
 """
 from ast import literal_eval
 
-
 import boto3
 from aws_xray_sdk.core import patch_all
+
+# ADF imports
 from logger import configure_logger
 
 patch_all()

src/lambda_codebase/account_processing/configure_account_tags.py

@@ -9,6 +9,8 @@
 """
 import boto3
 from aws_xray_sdk.core import patch_all
+
+# ADF imports
 from logger import configure_logger
 
 patch_all()

src/lambda_codebase/account_processing/create_account.py

@@ -8,6 +8,8 @@
 import os
 from aws_xray_sdk.core import patch_all
 import boto3
+
+# ADF imports
 from logger import configure_logger
 
 patch_all()

src/lambda_codebase/account_processing/delete_default_vpc.py

@@ -5,9 +5,11 @@
 Deletes the default VPC in a particular region
 """
 import os
-from sts import STS
 from aws_xray_sdk.core import patch_all
+
+# ADF imports
 from logger import configure_logger
+from sts import STS
 
 patch_all()
 

src/lambda_codebase/account_processing/get_account_regions.py

@@ -6,9 +6,11 @@
 """
 
 import os
-from sts import STS
 from aws_xray_sdk.core import patch_all
+
+# ADF imports
 from logger import configure_logger
+from sts import STS
 
 patch_all()
 

src/lambda_codebase/account_processing/process_account_files.py

@@ -13,15 +13,17 @@
 import logging
 from typing import Any, TypedDict
 import re
-import yaml
 
+import yaml
 from yaml.error import YAMLError
 
 import boto3
 from botocore.exceptions import ClientError
 from aws_xray_sdk.core import patch_all
-from organizations import Organizations
+
+# ADF imports
 from logger import configure_logger
+from organizations import Organizations
 
 
 patch_all()

src/lambda_codebase/account_processing/register_account_for_support.py

@@ -11,9 +11,11 @@
 import boto3
 from botocore.exceptions import ClientError, BotoCoreError
 from botocore.config import Config
-from logger import configure_logger
 from aws_xray_sdk.core import patch_all
 
+# ADF imports
+from logger import configure_logger
+
 
 LOGGER = configure_logger(__name__)
 patch_all()

src/lambda_codebase/cross_region_bucket/main.py

@@ -23,6 +23,7 @@
     delete,
 )
 
+# ADF imports
 from partition import get_partition
 
 # Type aliases:

src/lambda_codebase/deployment_account_config.py

@@ -14,6 +14,7 @@
 """
 
 import os
+
 import boto3
 
 from cloudformation import CloudFormation

src/lambda_codebase/determine_event.py

@@ -6,12 +6,14 @@
 """
 
 import os
+
 import boto3
 
-from parameter_store import ParameterStore
+# ADF imports
 from cache import Cache
 from event import Event
 from organizations import Organizations
+from parameter_store import ParameterStore
 
 REGION_DEFAULT = os.environ["AWS_REGION"]
 

src/lambda_codebase/event.py

@@ -8,13 +8,16 @@
 
 import ast
 import os
+
+# ADF imports
 from errors import ParameterNotFoundError, RootOUIDError
 
 DEPLOYMENT_ACCOUNT_OU_NAME = 'deployment'
 DEPLOYMENT_ACCOUNT_S3_BUCKET = os.environ["DEPLOYMENT_ACCOUNT_BUCKET"]
 ADF_VERSION = os.environ["ADF_VERSION"]
 ADF_LOG_LEVEL = os.environ["ADF_LOG_LEVEL"]
 
+
 class Event:
     """
     Class for structuring the Event in Step Functions
@@ -71,7 +74,6 @@ def __init__(self, event, parameter_store, organizations, account_id):
         )
         self.set_destination_ou_name()
 
-
     def _determine_if_deployment_account(self):
         """
         Sets property based on if the account that has been moved
@@ -126,7 +128,7 @@ def create_output_object(self, account_path):
             'full_path': "ROOT" if self.moved_to_root else account_path,
             'destination_ou_id': self.destination_ou_id,
             'source_ou_id': self.source_ou_id,
-            'deployment_account_parameters' : {
+            'deployment_account_parameters': {
                 'organization_id': organization_information.get(
                     "organization_id"
                 ),
@@ -139,5 +141,5 @@ def create_output_object(self, account_path):
                 'deployment_account_bucket': DEPLOYMENT_ACCOUNT_S3_BUCKET,
                 'adf_version': ADF_VERSION,
                 'adf_log_level': ADF_LOG_LEVEL
-            }
+            },
         }

src/lambda_codebase/generic_account_config.py

@@ -14,10 +14,11 @@
 
 import os
 
+# ADF imports
 from logger import configure_logger
-from sts import STS
-from stepfunctions import StepFunctions
 from partition import get_partition
+from stepfunctions import StepFunctions
+from sts import STS
 
 LOGGER = configure_logger(__name__)
 REGION_DEFAULT = os.getenv('AWS_REGION')

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml

@@ -72,26 +72,22 @@ Conditions:
 
 Globals:
   Function:
+    Architectures:
+      - arm64
     CodeUri: lambda_codebase
     Runtime: python3.12
 
 Resources:
-  LambdaLayerVersion:
+  ADFSharedPythonLambdaLayerVersion:
     Type: "AWS::Serverless::LayerVersion"
     Properties:
-      ContentUri: "../../adf-build/shared/"
+      ContentUri: "../../adf-build/shared/python"
       CompatibleRuntimes:
         - python3.12
       Description: "Shared Lambda Layer between master and deployment account"
-      LayerName: shared_layer
-
-  LambdaLayerVersionPermission:
-    Type: "AWS::Lambda::LayerVersionPermission"
-    Properties:
-      Action: lambda:GetLayerVersion
-      LayerVersionArn: !Ref LambdaLayerVersion
-      OrganizationId: !Ref OrganizationId
-      Principal: "*"
+      LayerName: adf_shared_layer
+    Metadata:
+      BuildMethod: python3.12
 
   KMSKey:
     Type: AWS::KMS::Key
@@ -186,7 +182,7 @@ Resources:
     Properties:
       Location: pipeline_management.yml
       Parameters:
-        LambdaLayer: !Ref LambdaLayerVersion
+        LambdaLayer: !Ref ADFSharedPythonLambdaLayerVersion
         ADFVersion: !Ref ADFVersion
         OrganizationId: !Ref OrganizationId
         CrossAccountAccessRole: !Ref CrossAccountAccessRole
@@ -1015,7 +1011,7 @@ Resources:
     Type: "AWS::Serverless::Function"
     Properties:
       Layers:
-        - !Ref LambdaLayerVersion
+        - !Ref ADFSharedPythonLambdaLayerVersion
       Description: "ADF Lambda Function - Send Slack Notification"
       FunctionName: SendSlackNotification
       Handler: slack.lambda_handler
@@ -1025,12 +1021,14 @@ Resources:
           ADF_PIPELINE_PREFIX: !Ref PipelinePrefix
           ADF_LOG_LEVEL: !Ref ADFLogLevel
       Timeout: 10
+    Metadata:
+      BuildMethod: python3.12
 
   EnableCrossAccountAccess:
     Type: "AWS::Serverless::Function"
     Properties:
       Layers:
-        - !Ref LambdaLayerVersion
+        - !Ref ADFSharedPythonLambdaLayerVersion
       Description: "ADF Lambda Function - EnableCrossAccountAccess"
       MemorySize: 1024
       Environment:
@@ -1043,12 +1041,14 @@ Resources:
       Handler: enable_cross_account_access.lambda_handler
       Role: !GetAtt EnableCrossAccountAccessLambdaRole.Arn
       Timeout: 900
+    Metadata:
+      BuildMethod: python3.12
 
   CheckPipelineStatus:
     Type: "AWS::Serverless::Function"
     Properties:
       Layers:
-        - !Ref LambdaLayerVersion
+        - !Ref ADFSharedPythonLambdaLayerVersion
       Description: "ADF Lambda Function - CheckPipelineStatus"
       Environment:
         Variables:
@@ -1059,6 +1059,8 @@ Resources:
       Handler: update_pipelines.lambda_handler
       Role: !GetAtt CheckPipelineStatusLambdaRole.Arn
       Timeout: 120
+    Metadata:
+      BuildMethod: python3.12
 
   SendSlackNotificationLambdaRole:
     Type: "AWS::IAM::Role"
@@ -1175,7 +1177,7 @@ Resources:
             Action:
               - "lambda:GetLayerVersion"
             Resource:
-              - !Ref LambdaLayerVersion
+              - !Ref ADFSharedPythonLambdaLayerVersion
           - Effect: Allow
             Action:
               - "logs:CreateLogGroup"
@@ -1384,6 +1386,8 @@ Resources:
                 - codecommit:GetRepository
               Resource: !GetAtt CodeCommitRepository.Arn
       FunctionName: ADFPipelinesDetermineDefaultBranchName
+    Metadata:
+      BuildMethod: python3.12
 
   InitialCommit:
     Type: Custom::InitialCommit
@@ -1416,6 +1420,8 @@ Resources:
               Resource: !GetAtt CodeCommitRepository.Arn
       FunctionName: PipelinesCreateInitialCommitFunction
       Timeout: 300
+    Metadata:
+      BuildMethod: python3.12
 
   KmsKeyArnParameter:
     Type: "AWS::SSM::Parameter"

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/enable_cross_account_access.py

@@ -9,14 +9,16 @@
 """
 
 import os
-import boto3
 
+import boto3
 from botocore.exceptions import ClientError
+
+# ADF imports
+from iam_cfn_deploy_role_policy import IAMCfnDeployRolePolicy
 from logger import configure_logger
 from parameter_store import ParameterStore
-from sts import STS
 from partition import get_partition
-from iam_cfn_deploy_role_policy import IAMCfnDeployRolePolicy
+from sts import STS
 
 
 KEY_ID = os.environ["KMS_KEY_ID"]

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/initial_commit/initial_commit.py

@@ -38,7 +38,7 @@
 PR_DESCRIPTION = """ADF Version {0}
 
 You can find the changelog at:
-https://github.com/awslabs/aws-deployment-framework/releases/tag/v{0}
+https://github.com/awslabs/aws-deployment-framework/releases/tag/{0}
 
 This PR was automatically created when you deployed version {0} of the
 AWS Deployment Framework.

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/slack.py

@@ -11,6 +11,7 @@
 import urllib
 import boto3
 
+# ADF imports
 from parameter_store import ParameterStore