Feat/CodeCommit provider - set "default-scm-codecommit-account-id" in…

… adfconfig.yml and set default fallback (#633)

* feat: set default scm codecommit account id

---------

Co-authored-by: Andreas Falkenberg
Co-authored-by: Simon Kok

docs/admin-guide.md

@@ -237,6 +237,15 @@ Config has five components in `main-notification-endpoint`, `scp`, `scm`,
     your buildspec. If this variable is not set, the SSM parameter
     `/adf/org/stage` defaults to "none".
 
+  - **default-scm-codecommit-account-id** allows you to configure the default
+    account id that should be used with all source-code management platforms
+    that ADF supports.
+    If not set here, the deployment account id is taken as default value.
+    The CodeCommit account-id can be still be overwritten with an explicit
+    account id in the individual deployment map.
+    The CodeCommit provider guide provides more details:
+    [providers-guide.md.yml: CodeCommit](./providers-guide.md#codecommit).
+
 ## Accounts
 
 ### Management account

docs/providers-guide.md

@@ -66,11 +66,15 @@ Provider type: `codecommit`.
 
 #### Properties
 
-- *account_id* - *(String)* **(required)**
+- *account_id* - *(String)* **(optional)**
   - The AWS Account ID where the Source Repository is located. If the repository
     does not exist it will be created via AWS CloudFormation on the source
     account along with the associated cross account CloudWatch event action to
     trigger the pipeline.
+  - Additionally, the default account id for CodeCommit, can be set in
+    [adfconfig.yml: config/scm/default-scm-codecommit-account-id](./admin-guide.md#adfconfig).
+  - If not set here in the provider and if not set in adfconfig.yml,
+    the deployment account id will be used as default value.
 - *repository* - *(String)* defaults to name of the pipeline.
   - The AWS CodeCommit repository name.
 - *branch* - *(String)* default to configured [adfconfig.yml: config/scm/default-scm-branch](./admin-guide.md#adfconfig).

src/lambda_codebase/initial_commit/adfconfig.yml.j2

@@ -27,9 +27,12 @@ config:
   scm:
     auto-create-repositories: enabled
     default-scm-branch: main
+    # Optional:
+    # default-scm-codecommit-account-id: "123456789012"
   deployment-maps:
     allow-empty-target: "False"
     # ^ Needs to be "True" or "False". Defaults to "False" when not set.
   #org:
     # Optional: Use this variable to define the AWS Organization in case of staged multi-organization ADF deployments
     #stage: dev
+

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py

@@ -11,10 +11,12 @@
 from rule import Rule
 from logger import configure_logger
 from cloudwatch import ADFMetrics
-
+from errors import ParameterNotFoundError
+from parameter_store import ParameterStore
 
 LOGGER = configure_logger(__name__)
 DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"]
+DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"]
 CLOUDWATCH = boto3.client("cloudwatch")
 METRICS = ADFMetrics(CLOUDWATCH, "PIPELINE_MANAGEMENT/RULE")
 
@@ -58,6 +60,40 @@ def lambda_handler(event, _):
         .get("properties", {})
         .get("account_id")
     )
+
+    # Resolve codecommit source_account_id in case it is not set
+    if source_provider == "codecommit" and not source_account_id:
+        # Evaluate as follows:
+        # If not set, we have to set it with
+        #   - default_scm_codecommit_account_id (if exists)
+        #   - or ADF_DEPLOYMENT_ACCOUNT_ID
+        # If set, we are done anyways
+        LOGGER.debug(
+            "source_account_id not found in source_props - ADF will set "
+            "it from SSM param default_scm_codecommit_account_id.",
+        )
+        deployment_account_id = DEPLOYMENT_ACCOUNT_ID
+        try:
+            parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3)
+            default_scm_codecommit_account_id = parameter_store.fetch_parameter(
+                "/adf/scm/default-scm-codecommit-account-id",
+            )
+        except ParameterNotFoundError:
+            default_scm_codecommit_account_id = deployment_account_id
+            LOGGER.debug(
+                "default_scm_codecommit_account_id not found in SSM - "
+                "Fall back to deployment_account_id.",
+            )
+        source_account_id = default_scm_codecommit_account_id
+
+        # Create the properties object if it does not exist
+        if pipeline["default_providers"]["source"].get("properties") is None:
+            pipeline["default_providers"]["source"]["properties"] = {}
+
+        pipeline["default_providers"]["source"]["properties"]["account_id"] = (
+            source_account_id
+        )
+
     if (
         source_provider == "codecommit"
         and source_account_id

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py

@@ -67,6 +67,7 @@ def lambda_handler(event, _):
         .get("properties", {})
         .get("repository", {})
     )
+
     if (
         code_account_id
         and str(code_account_id).isdigit()

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/generate_pipeline_inputs.py

@@ -72,6 +72,9 @@ def fetch_required_ssm_params(pipeline_input, regions):
             output["default_scm_branch"] = parameter_store.fetch_parameter(
                 "default_scm_branch",
             )
+            output["default_scm_codecommit_account_id"] = parameter_store.fetch_parameter(
+                "/adf/scm/default-scm-codecommit-account-id",
+            )
             codestar_connection_path = (
                 pipeline_input
                 .get("default_providers", {})
@@ -172,8 +175,13 @@ def generate_pipeline_inputs(
         data["pipeline_input"]["default_providers"]["source"]["properties"][
             "codestar_connection_arn"
         ] = data["ssm_params"]["codestar_connection_arn"]
-    data["pipeline_input"]["default_scm_branch"] = data["ssm_params"].get(
-        "default_scm_branch",
+    data["pipeline_input"]["default_scm_branch"] = (
+        data["ssm_params"]
+        .get("default_scm_branch")
+    )
+    data["pipeline_input"]["default_scm_codecommit_account_id"] = (
+        data["ssm_params"]
+        .get("default_scm_codecommit_account_id")
     )
     store_regional_parameter_config(
         pipeline_object,

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml

@@ -160,6 +160,18 @@ Resources:
                 - "lambda.amazonaws.com"
             Action:
               - "sts:AssumeRole"
+      Policies:
+        - PolicyName: "adf-pipeline-create-update-rule-policy"
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Effect: Allow
+                Action:
+                  - "ssm:GetParameter"
+                  - "ssm:GetParameters"
+                  - "ssm:GetParametersByPath"
+                Resource:
+                  - !Sub arn:${AWS::Partition}:ssm:*:*:parameter/adf/*
 
   CreateRepositoryLambdaRole:
     Type: "AWS::IAM::Role"

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py

@@ -148,9 +148,18 @@ def prepare_deployment_account(sts, deployment_account_id, config):
     )
     deployment_account_parameter_store.put_parameter(
         'default_scm_branch',
-        config.config.get('scm', {}).get(
-            'default-scm-branch',
-            ADF_DEFAULT_SCM_FALLBACK_BRANCH,
+        (
+            config.config
+            .get('scm', {})
+            .get('default-scm-branch', ADF_DEFAULT_SCM_FALLBACK_BRANCH)
+        )
+    )
+    deployment_account_parameter_store.put_parameter(
+        '/adf/scm/default-scm-codecommit-account-id',
+        (
+            config.config
+            .get('scm', {})
+            .get('default-scm-codecommit-account-id', deployment_account_id)
         )
     )
     deployment_account_parameter_store.put_parameter(

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codecommit.py

@@ -23,6 +23,7 @@ def __init__(self, scope: Construct, id: str, map_params: dict, **kwargs): #pyli
         default_providers = map_params.get("default_providers", {})
         source_props = default_providers.get("source", {}).get("properties", {})
         account_id = source_props.get("account_id", ADF_DEPLOYMENT_ACCOUNT_ID)
+
         self.source = _codepipeline.CfnPipeline.StageDeclarationProperty(
             name=f"Source-{account_id}",
             actions=[

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py

@@ -25,6 +25,7 @@
 ADF_PIPELINE_PREFIX = os.environ.get("ADF_PIPELINE_PREFIX", "")
 ADF_DEFAULT_BUILD_TIMEOUT = 20
 ADF_DEFAULT_SCM_FALLBACK_BRANCH = 'master'
+ADF_DEFAULT_SCM_CODECOMMIT_ACCOUNT_ID = os.environ["ACCOUNT_ID"]
 
 
 LOGGER = configure_logger(__name__)
@@ -74,6 +75,10 @@ def __init__(self, **kwargs):
             "default_scm_branch",
             ADF_DEFAULT_SCM_FALLBACK_BRANCH,
         )
+        self.default_scm_codecommit_account_id = self.map_params.get(
+            "/adf/scm/default_scm_codecommit_account_id",
+            ADF_DEFAULT_SCM_CODECOMMIT_ACCOUNT_ID,
+        )
         self.configuration = self._generate_configuration()
         self.config = self.generate()
 
@@ -731,6 +736,10 @@ def __init__(
             "default_scm_branch",
             ADF_DEFAULT_SCM_FALLBACK_BRANCH,
         )
+        self.default_scm_codecommit_account_id = map_params.get(
+            "/adf/scm/default_scm_codecommit_account_id",
+            ADF_DEFAULT_SCM_CODECOMMIT_ACCOUNT_ID,
+        )
         self.cfn = _codepipeline.CfnPipeline(
             self,
             'pipeline',

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/schema_validation.py

@@ -70,7 +70,7 @@
 }
 CODECOMMIT_SOURCE = {
     "provider": 'codecommit',
-    "properties": CODECOMMIT_SOURCE_PROPS
+    Optional("properties"): CODECOMMIT_SOURCE_PROPS,
 }
 
 # GitHub Source
@@ -284,13 +284,23 @@
     'codebuild': Schema(DEFAULT_CODEBUILD_BUILD),
 }
 PROVIDER_SCHEMA = {
-    'source': And(
-        {
-            'provider': Or('codecommit', 'github', 's3', 'codestar'),
-            'properties': dict,
-        },
-        # pylint: disable=W0108
-        lambda x: PROVIDER_SOURCE_SCHEMAS[x['provider']].validate(x),
+    'source': Or(
+        And(
+            {
+                'provider': Or('github', 's3', 'codestar'),
+                'properties': dict,
+            },
+            # pylint: disable=W0108
+            lambda x: PROVIDER_SOURCE_SCHEMAS[x['provider']].validate(x),
+        ),
+        And(
+            {
+                'provider': Or('codecommit'),
+                Optional('properties'): dict,
+            },
+            # pylint: disable=W0108
+            lambda x: PROVIDER_SOURCE_SCHEMAS[x['provider']].validate(x),
+        ),
     ),
     Optional('build'): And(
         {

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_main.py

@@ -133,7 +133,7 @@ def test_prepare_deployment_account_defaults(param_store_cls, cls, sts):
     )
     for param_store in parameter_store_list:
         assert param_store.put_parameter.call_count == (
-            12 if param_store == deploy_param_store else 2
+            13 if param_store == deploy_param_store else 2
         )
         param_store.put_parameter.assert_has_calls(
             [
@@ -225,7 +225,7 @@ def test_prepare_deployment_account_specific_config(param_store_cls, cls, sts):
     )
     for param_store in parameter_store_list:
         assert param_store.put_parameter.call_count == (
-            14 if param_store == deploy_param_store else 2
+            15 if param_store == deploy_param_store else 2
         )
         param_store.put_parameter.assert_has_calls(
             [