diff --git a/.github/workflows/codeartifact-build.yml b/.github/workflows/codeartifact-build.yml new file mode 100644 index 00000000..62b9e128 --- /dev/null +++ b/.github/workflows/codeartifact-build.yml @@ -0,0 +1,50 @@ +name: CodeArtifact - Build +run-name: Test IaC @ ${{ github.ref_name }} + +on: + push: + paths: + - .github/workflows/codeartifact-build.yml + - CodeArtifact/cdk/** + workflow_dispatch: + +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }} + +defaults: + run: + shell: bash + +jobs: + central-stack: + name: Test central resources IaC + runs-on: ubuntu-latest + defaults: + run: + working-directory: CodeArtifact/cdk/central_resources + env: + ENV_STAGE: dev + steps: + - uses: actions/checkout@v4 + - run: make lint-python + - uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Set up aws-cdk + run: make install-cdk + - name: Print deployment environment + run: | + echo "INFO: cdk version: $(cdk --version)" + echo "INFO: node version: $(node --version)" + echo "INFO: npm version: $(npm --version)" + echo "INFO: python3 version: $(python3 --version)" + + - name: Run cdk synth + run: make synth + + - name: Run cdk-validator-cfnguard + env: + ENV: ${{ needs.common.outputs.environment }} + run: | + make test-with-cdk-validator-cfnguard diff --git a/CHANGELOG.md b/CHANGELOG.md index 668d87ab..5d3e44da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,10 +3,17 @@ All notable changes to this project will be documented in this file. +## 2024-10-05 + +### Added + * [CodeArtifact/cdk/central_resources/](CodeArtifact/cdk/central_resources/) cdk and workflow for deploying a central_resources for hosting CodeArtifact external connections and a shared CodeArtifact repository, + * Added `black` configurations in [pyproject.toml](./pyproject.toml). + * Added `flake8` configurations in [setup.cfg](./setup.cfg). + ## 2024-10-03 ### Added - * Added [DynamoDB/export_ddb_to_s3.py](DynamoDB/export_ddb_to_s3.py), which exports a DynamoDB table to S3 bucket, then downloads the exported data from S3, unzips the files, and merges the data into a single JSON file, then upload back to S3. + * Added [DynamoDB/export_ddb_to_s3.py](DynamoDB/export_ddb_to_s3.py), which exports a DynamoDB table to S3 bucket, then downloads the exported data from S3, unzips the files, and merges the data into a single JSON file, then upload back to S3. ## 2024-10-02 diff --git a/CodeArtifact/cdk/central_resources/Makefile b/CodeArtifact/cdk/central_resources/Makefile new file mode 100644 index 00000000..246fe4f6 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/Makefile @@ -0,0 +1,39 @@ +export AWS_DEFAULT_REGION ?= ap-southeast-2 +export CDK_DEFAULT_REGION ?= ap-southeast-2 +export ENV_STAGE ?= dev + +APP_NAME=$(shell grep -m 1 AppName environment/$(ENV_STAGE).yml | cut -c 10-) + +install-cdk: + npm install -g aws-cdk + python3 -m pip install -U pip + pip3 install -r requirements-dev.txt + +synth: + cdk synth -c env=${ENV_STAGE} --all + +deploy: + pip3 install -r requirements.txt + cdk deploy $(APP_NAME) -c env=${ENV_STAGE} --require-approval never + +destroy: + cdk destroy $(APP_NAME) -f -c env=${ENV_STAGE} + +test-cdk: + python3 -m pytest tests/ + +test-with-cdk-validator-cfnguard: synth + +pre-commit: format-python lint-python lint-yaml test + +format-python: + black **.py */**.py + +lint-python: + flake8 **.py */**.py + +lint-yaml: + yamllint -c .github/linters/.yaml-lint.yml -f parsable . + +clean: + rm -rf cdk.out **/__pycache__ diff --git a/CodeArtifact/cdk/central_resources/app.py b/CodeArtifact/cdk/central_resources/app.py new file mode 100644 index 00000000..a0a9ead4 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/app.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +from os.path import ( + dirname, + join, + realpath, +) + +import yaml +from aws_cdk import ( + App, + CliCredentialsStackSynthesizer, + Environment, + Tags, +) +from cdklabs.cdk_validator_cfnguard import ( + CfnGuardValidator, +) +from lib.central_resources import ( + CodeArtifactCentralResources, +) + +ENV_DIR = join( + dirname(realpath(__file__)), + "environment", +) + + +def main(): + app = App( + policy_validation_beta1=[ + CfnGuardValidator( + # By default the CfnGuardValidator plugin has the Control Tower proactive + # rules enabled. If you wish to disable them, set this to false. + control_tower_rules_enabled=True, + # You can also disable individual rules by passing in a list of rule names + # e.g. "ct-s3-pr-1" list is https://github.com/cdklabs/cdk-validator-cfnguard + disabled_rules=[], + # You can also pass in a list of local guard files or directory paths + # e.g. "./guards", "./guards/custom-guard-file.guard" + rules=[], + ) + ] + ) + + ENV_NAME = app.node.try_get_context("env") or "dev" + + with open( + join( + ENV_DIR, + f"{ENV_NAME}.yaml", + ), + "r", + ) as stream: + yaml_data = yaml.safe_load(stream) + config = yaml_data if yaml_data is not None else {} + + stack = CodeArtifactCentralResources( + scope=app, + id="CodeArtifactCentralResources", + config=config, + env=Environment( + account=config["AWS_ACCOUNT"], + region=config["AWS_REGION"], + ), + synthesizer=CliCredentialsStackSynthesizer(), + termination_protection=(ENV_NAME == "prd"), + ) + + # Add common tags + for key, value in config["TAGS"].items(): + Tags.of(stack).add(key, value) + + Tags.of(stack).add( + "Description", + "CodeArtifact central resources", + ) + Tags.of(stack).add( + "Environment", + ENV_NAME, + ) + Tags.of(stack).add( + "Name", + stack.stack_name, + ) + + app.synth() + + +if __name__ == "__main__": + main() diff --git a/CodeArtifact/cdk/central_resources/cdk.json b/CodeArtifact/cdk/central_resources/cdk.json new file mode 100644 index 00000000..dfda3b70 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/cdk.json @@ -0,0 +1,5 @@ +{ + "app": "python3 app.py", + "context": { + } +} diff --git a/CodeArtifact/cdk/central_resources/environment/dev.yaml b/CodeArtifact/cdk/central_resources/environment/dev.yaml new file mode 100644 index 00000000..cbf50a97 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/environment/dev.yaml @@ -0,0 +1,25 @@ +AppName: CodeArtifactCentralResources +AWS_ACCOUNT: "111111111111" +AWS_ORG_ID: o-12324567890 +AWS_REGION: ap-southeast-2 +DOMAIN_NAME: todo-dev +EXTERNAL_CONNECTIONS: + - crates-io + - maven-central + - maven-clojars + - maven-commonsware + - maven-googleandroid + - maven-gradleplugins + - npmjs + - nuget-org + - pypi + - ruby-gems-org +INTERNAL_SHARED_REPO: internal-shared-dev +KEY_ADMIN_ARNS: + - arn:aws:iam::111111111111:role/key-admin + - arn:aws:iam::111111111111:role/deploy-role +WRITE_ROLE_ARNS_LIKE: + - arn:aws:iam::222222222222:role/*/deploy-role +TAGS: + CostCentre: TODO + Project: TODO diff --git a/CodeArtifact/cdk/central_resources/lib/central_resources.py b/CodeArtifact/cdk/central_resources/lib/central_resources.py new file mode 100644 index 00000000..7660fb76 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/lib/central_resources.py @@ -0,0 +1,228 @@ +from aws_cdk import ( + CfnOutput, + Stack, + aws_iam, + aws_kms, +) +from aws_cdk.aws_codeartifact import ( + CfnDomain, + CfnRepository, +) +from constructs import ( + Construct, +) +from lib.kms import ( + create_kms_key_and_alias, +) + + +class CodeArtifactCentralResources(Stack): + def __init__( + self, + scope: Construct, + id: str, + config: dict, + **kwargs, + ) -> None: + super().__init__( + scope, + id, + **kwargs, + ) + + key_admin_arns = [ + aws_iam.Role.from_role_arn( + self, + arn_str.split("/")[-1], + arn_str, + ) + for arn_str in config["KEY_ADMIN_ARNS"] + ] + + key_alias = create_kms_key_and_alias( + self, + "CodeArtifactKey", + key_alias=f"alias/codeartifact-{config['DOMAIN_NAME']}", + key_admin_arns=key_admin_arns, + ) + + # Create CodeArtifact domain + domain = self.create_domain_with_domain_policy( + config["DOMAIN_NAME"], + config["AWS_ORG_ID"], + key_alias, + ) + domain.add_dependency(key_alias.node.default_child) # casting L1/L2 construct + + read_only_statement = self.create_read_policy_statament(config["AWS_ORG_ID"]) + + # Create external connection repos + external_conn_repos = [ + self.create_external_connection_repo( + domain, + external_conn, + read_only_statement, + ) + for external_conn in config["EXTERNAL_CONNECTIONS"] + ] + + write_statement = self.create_write_policy_statament(config["WRITE_ROLE_ARNS_LIKE"]) + + # Create internal shared repos with upstreams as the external connection repos + internal_shared_repo = self.create_internal_shared_repo( + domain, + config["INTERNAL_SHARED_REPO"], + external_conn_repos, + read_only_statement, + write_statement, + ) + + CfnOutput( + self, + "CodeArtifactDomainName", + value=domain.domain_name, + ) + CfnOutput( + self, + "CodeArtifactDomainOwner", + value=self.account, + ) + CfnOutput( + self, + "InternalSharedRepoName", + value=internal_shared_repo.repository_name, + ) + + def create_domain_with_domain_policy( + self, + domain_name: str, + org_id: str, + key_alias: aws_kms.Alias, + ) -> CfnDomain: + return CfnDomain( + self, + "CodeArtifactDomain", + domain_name=domain_name, + encryption_key=key_alias.key_arn, + permissions_policy_document=aws_iam.PolicyDocument( + statements=[ + aws_iam.PolicyStatement( + actions=[ + "codeartifact:CreateRepository", + "codeartifact:DescribeDomain", + "codeartifact:GetAuthorizationToken", + "codeartifact:GetDomainPermissionsPolicy", + "codeartifact:ListRepositoriesInDomain", + ], + conditions={"StringEquals": {"aws:PrincipalOrgId": [org_id]}}, + effect=aws_iam.Effect.ALLOW, + principals=[aws_iam.AnyPrincipal()], + resources=["*"], + sid="DomainPolicyForOrganization", + ), + ] + ), + ) + + def create_external_connection_repo( + self, + domain: CfnDomain, + external_conn: str, + read_only_statement: aws_iam.PolicyStatement, + ) -> CfnRepository: + repo = CfnRepository( + self, + f"ExternalConnection{external_conn.capitalize()}", + description=f"External connection {external_conn}", + domain_name=domain.domain_name, + domain_owner=self.account, + external_connections=[f"public:{external_conn}"], + permissions_policy_document=aws_iam.PolicyDocument(statements=[read_only_statement]), + repository_name=f"external-{external_conn}", + ) + repo.add_dependency(domain) + return repo + + def create_internal_shared_repo( + self, + domain: CfnDomain, + internal_shared_repo: str, + external_conn_repos: list, + read_only_statement: aws_iam.PolicyStatement, + write_statement: aws_iam.PolicyStatement, + ) -> CfnRepository: + external_conn_repo_names = [repo.repository_name for repo in external_conn_repos] + + repo = CfnRepository( + self, + "InternalSharedRepo", + description="Internal shared", + domain_name=domain.domain_name, + domain_owner=self.account, + permissions_policy_document=aws_iam.PolicyDocument( + statements=[ + read_only_statement, + write_statement, + ] + ), + repository_name=internal_shared_repo, + upstreams=external_conn_repo_names, + ) + + repo.add_dependency(domain) + for external_conn_repo in external_conn_repos: + repo.add_dependency(external_conn_repo) + + return repo + + def create_read_policy_statament(self, org_id: str): + return aws_iam.PolicyStatement( + actions=[ + "codeartifact:AssociateWithDownstreamRepository", + "codeartifact:DescribePackageVersion", + "codeartifact:DescribeRepository", + "codeartifact:GetPackageVersionAsset", + "codeartifact:GetPackageVersionReadme", + "codeartifact:GetRepositoryEndpoint", + "codeartifact:GetRepositoryPermissionsPolicy", + "codeartifact:ListPackages", + "codeartifact:ListPackageVersions", + "codeartifact:ListPackageVersionAssets", + "codeartifact:ListPackageVersionDependencies", + "codeartifact:ListTagsForResource", + "codeartifact:ReadFromRepository", + ], + conditions={"StringEquals": {"aws:PrincipalOrgId": [org_id]}}, + effect=aws_iam.Effect.ALLOW, + principals=[aws_iam.AnyPrincipal()], + resources=["*"], + sid="ReadAccess", + ) + + def create_write_policy_statament( + self, + write_role_arn_like_list: list, + ): + """ + Note: WriteAccess is on request - add to `conditions` and `principal` below + """ + + account_ids = [arn.split(":")[4] for arn in write_role_arn_like_list] + root_principals = [ + aws_iam.ArnPrincipal(f"arn:aws:iam::{account_id}:root") for account_id in account_ids + ] + + return aws_iam.PolicyStatement( + actions=[ + "codeartifact:CopyPackageVersions", + "codeartifact:DeletePackage", + "codeartifact:DeletePackageVersions", + "codeartifact:PublishPackageVersion", + "codeartifact:PutPackageMetadata", + ], + conditions={"ArnLike": {"aws:PrincipalArn": write_role_arn_like_list}}, + effect=aws_iam.Effect.ALLOW, + principals=root_principals, + resources=["*"], + sid="WriteAccess", + ) diff --git a/CodeArtifact/cdk/central_resources/lib/kms.py b/CodeArtifact/cdk/central_resources/lib/kms.py new file mode 100644 index 00000000..a3de5005 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/lib/kms.py @@ -0,0 +1,59 @@ +from aws_cdk import ( + aws_iam, + aws_kms, +) +from constructs import ( + Construct, +) + + +def create_kms_key_and_alias( + scope: Construct, + construct_id: str, + key_alias: str, + key_admin_arns, +) -> aws_kms.Key: + policy_document = aws_iam.PolicyDocument( + statements=[ + aws_iam.PolicyStatement( + actions=["kms:*"], + principals=[aws_iam.AccountRootPrincipal()], + resources=["*"], + sid="EnableIAMUserPermissions", + ), + aws_iam.PolicyStatement( + actions=[ + "kms:Create*", + "kms:Describe*", + "kms:Enable*", + "kms:List*", + "kms:Put*", + "kms:Update*", + "kms:Revoke*", + "kms:Disable*", + "kms:Get*", + "kms:Delete*", + "kms:TagResource", + "kms:UntagResource", + "kms:ScheduleKeyDeletion", + "kms:CancelKeyDeletion", + ], + principals=key_admin_arns, + resources=["*"], + sid="AllowAccessForKeyAdministrators", + ), + ] + ) + + key = aws_kms.Key( + scope, + construct_id, + enable_key_rotation=True, + policy=policy_document, + ) + return aws_kms.Alias( + scope, + f"{construct_id}Alias", + alias_name=key_alias, + target_key=key, + ) diff --git a/CodeArtifact/cdk/central_resources/requirements-dev.txt b/CodeArtifact/cdk/central_resources/requirements-dev.txt new file mode 100644 index 00000000..ecb929a6 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/requirements-dev.txt @@ -0,0 +1,6 @@ +-r requirements.txt +black~=23.7 +boto3~=1.28 +cdk-nag~=2.27 +flake8~=6.0 +pytest~=7.4 diff --git a/CodeArtifact/cdk/central_resources/requirements.txt b/CodeArtifact/cdk/central_resources/requirements.txt new file mode 100644 index 00000000..be4706a1 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/requirements.txt @@ -0,0 +1,4 @@ +aws-cdk-lib==2.158.0 +cdklabs.cdk-validator-cfnguard==0.0.58 +constructs==10.3.0 +pyyaml==6.0.1 \ No newline at end of file diff --git a/CodeArtifact/cdk/central_resources/tests-cdk/test_central_resources.py b/CodeArtifact/cdk/central_resources/tests-cdk/test_central_resources.py new file mode 100644 index 00000000..47ecfe6b --- /dev/null +++ b/CodeArtifact/cdk/central_resources/tests-cdk/test_central_resources.py @@ -0,0 +1,419 @@ +from os.path import ( + dirname, + join, + realpath, +) + +import pytest +import yaml +from aws_cdk import ( + App, + Aspects, + Environment, +) +from aws_cdk.assertions import ( + Template, +) +from cdk_nag import ( + AwsSolutionsChecks, + HIPAASecurityChecks, +) +from lib.central_resources import ( + CodeArtifactCentralResources, +) + +ENV_DIR = join( + dirname(dirname(dirname(realpath(__file__)))), + "environment", +) + + +@pytest.mark.parametrize( + ("env"), + [("dev"), ("prd")], +) +def test_synthesizes_properly( + env, +): + with open( + join( + ENV_DIR, + f"{env}.yaml", + ), + "r", + ) as stream: + yaml_data = yaml.safe_load(stream) + config = yaml_data if yaml_data is not None else {} + + app = App() + + _stack = CodeArtifactCentralResources( + app, + "TestCodeArtifactCentralResources", + config=config, + env=Environment( + account=config["AWS_ACCOUNT"], + region=config["AWS_REGION"], + ), + termination_protection=True, + ) + + # Add AWS Solutions and HIPAA Security checks + Aspects.of(app).add(AwsSolutionsChecks(verbose=True)) + Aspects.of(app).add(HIPAASecurityChecks(verbose=True)) + + # Prepare the stack for assertions. + template = Template.from_stack(_stack) + + template.resource_count_is( + "AWS::CodeArtifact::Domain", + 1, + ) + + ################################################################################ + # Check kms key + + # Assert it creates the function with the correct properties. + template.has_resource_properties( + "AWS::KMS::Alias", + { + "AliasName": f"alias/codeartifact-{config['DOMAIN_NAME']}", + }, + ) + + template.has_resource_properties( + "AWS::KMS::Key", + { + "EnableKeyRotation": True, + "KeyPolicy": { + "Statement": [ + { + "Action": "kms:*", + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::Join": [ + "", + [ + "arn:", + {"Ref": "AWS::Partition"}, + f":iam::{config['AWS_ACCOUNT']}:root", + ], + ] + } + }, + "Resource": "*", + "Sid": "EnableIAMUserPermissions", + }, + { + "Action": [ + "kms:Create*", + "kms:Describe*", + "kms:Enable*", + "kms:List*", + "kms:Put*", + "kms:Update*", + "kms:Revoke*", + "kms:Disable*", + "kms:Get*", + "kms:Delete*", + "kms:TagResource", + "kms:UntagResource", + "kms:ScheduleKeyDeletion", + "kms:CancelKeyDeletion", + ], + "Effect": "Allow", + "Principal": {"AWS": config["KEY_ADMIN_ARNS"]}, + "Resource": "*", + "Sid": "AllowAccessForKeyAdministrators", + }, + ] + }, + }, + ) + + ################################################################################ + # Check domain + + # Assert it creates the function with the correct properties. + template.has_resource_properties( + "AWS::CodeArtifact::Domain", + { + "DomainName": config["DOMAIN_NAME"], + "EncryptionKey": { + "Fn::Join": [ + "", + [ + "arn:", + {"Ref": "AWS::Partition"}, + f":kms:ap-southeast-2:{config['AWS_ACCOUNT']}:alias/codeartifact-{config['DOMAIN_NAME']}", + ], + ] + }, + "PermissionsPolicyDocument": { + "Statement": [ + { + "Condition": { + "StringEquals": {"aws:PrincipalOrgId": [config["AWS_ORG_ID"]]} + }, + "Effect": "Allow", + } + ] + }, + }, + ) + + # Check number of repos - 10 external and 1 internal + template.resource_count_is( + "AWS::CodeArtifact::Repository", + 11, + ) + + ################################################################################ + # Check external connection repos + + template.has_resource_properties( + "AWS::CodeArtifact::Repository", + { + "DomainName": config["DOMAIN_NAME"], + "DomainOwner": config["AWS_ACCOUNT"], + "ExternalConnections": ["public:crates-io"], + "PermissionsPolicyDocument": { + "Statement": [ + { + "Condition": { + "StringEquals": {"aws:PrincipalOrgId": [config["AWS_ORG_ID"]]} + }, + "Effect": "Allow", + } + ] + }, + "RepositoryName": "external-crates-io", + }, + ) + + template.has_resource_properties( + "AWS::CodeArtifact::Repository", + { + "DomainName": config["DOMAIN_NAME"], + "DomainOwner": config["AWS_ACCOUNT"], + "ExternalConnections": ["public:maven-central"], + "PermissionsPolicyDocument": { + "Statement": [ + { + "Condition": { + "StringEquals": {"aws:PrincipalOrgId": [config["AWS_ORG_ID"]]} + }, + "Effect": "Allow", + } + ] + }, + "RepositoryName": "external-maven-central", + }, + ) + + template.has_resource_properties( + "AWS::CodeArtifact::Repository", + { + "DomainName": config["DOMAIN_NAME"], + "DomainOwner": config["AWS_ACCOUNT"], + "ExternalConnections": ["public:maven-clojars"], + "PermissionsPolicyDocument": { + "Statement": [ + { + "Condition": { + "StringEquals": {"aws:PrincipalOrgId": [config["AWS_ORG_ID"]]} + }, + "Effect": "Allow", + } + ] + }, + "RepositoryName": "external-maven-clojars", + }, + ) + template.has_resource_properties( + "AWS::CodeArtifact::Repository", + { + "DomainName": config["DOMAIN_NAME"], + "DomainOwner": config["AWS_ACCOUNT"], + "ExternalConnections": ["public:maven-commonsware"], + "PermissionsPolicyDocument": { + "Statement": [ + { + "Condition": { + "StringEquals": {"aws:PrincipalOrgId": [config["AWS_ORG_ID"]]} + }, + "Effect": "Allow", + } + ] + }, + "RepositoryName": "external-maven-commonsware", + }, + ) + template.has_resource_properties( + "AWS::CodeArtifact::Repository", + { + "DomainName": config["DOMAIN_NAME"], + "DomainOwner": config["AWS_ACCOUNT"], + "ExternalConnections": ["public:maven-googleandroid"], + "PermissionsPolicyDocument": { + "Statement": [ + { + "Condition": { + "StringEquals": {"aws:PrincipalOrgId": [config["AWS_ORG_ID"]]} + }, + "Effect": "Allow", + } + ] + }, + "RepositoryName": "external-maven-googleandroid", + }, + ) + template.has_resource_properties( + "AWS::CodeArtifact::Repository", + { + "DomainName": config["DOMAIN_NAME"], + "DomainOwner": config["AWS_ACCOUNT"], + "ExternalConnections": ["public:maven-gradleplugins"], + "PermissionsPolicyDocument": { + "Statement": [ + { + "Condition": { + "StringEquals": {"aws:PrincipalOrgId": [config["AWS_ORG_ID"]]} + }, + "Effect": "Allow", + } + ] + }, + "RepositoryName": "external-maven-gradleplugins", + }, + ) + template.has_resource_properties( + "AWS::CodeArtifact::Repository", + { + "DomainName": config["DOMAIN_NAME"], + "DomainOwner": config["AWS_ACCOUNT"], + "ExternalConnections": ["public:npmjs"], + "PermissionsPolicyDocument": { + "Statement": [ + { + "Condition": { + "StringEquals": {"aws:PrincipalOrgId": [config["AWS_ORG_ID"]]} + }, + "Effect": "Allow", + } + ] + }, + "RepositoryName": "external-npmjs", + }, + ) + template.has_resource_properties( + "AWS::CodeArtifact::Repository", + { + "DomainName": config["DOMAIN_NAME"], + "DomainOwner": config["AWS_ACCOUNT"], + "ExternalConnections": ["public:nuget-org"], + "PermissionsPolicyDocument": { + "Statement": [ + { + "Condition": { + "StringEquals": {"aws:PrincipalOrgId": [config["AWS_ORG_ID"]]} + }, + "Effect": "Allow", + } + ] + }, + "RepositoryName": "external-nuget-org", + }, + ) + template.has_resource_properties( + "AWS::CodeArtifact::Repository", + { + "DomainName": config["DOMAIN_NAME"], + "DomainOwner": config["AWS_ACCOUNT"], + "ExternalConnections": ["public:pypi"], + "PermissionsPolicyDocument": { + "Statement": [ + { + "Condition": { + "StringEquals": {"aws:PrincipalOrgId": [config["AWS_ORG_ID"]]} + }, + "Effect": "Allow", + } + ] + }, + "RepositoryName": "external-pypi", + }, + ) + template.has_resource_properties( + "AWS::CodeArtifact::Repository", + { + "DomainName": config["DOMAIN_NAME"], + "DomainOwner": config["AWS_ACCOUNT"], + "ExternalConnections": ["public:ruby-gems-org"], + "PermissionsPolicyDocument": { + "Statement": [ + { + "Condition": { + "StringEquals": {"aws:PrincipalOrgId": [config["AWS_ORG_ID"]]} + }, + "Effect": "Allow", + } + ] + }, + "RepositoryName": "external-ruby-gems-org", + }, + ) + + ################################################################################ + # Check internal shared repo + + template.has_resource_properties( + "AWS::CodeArtifact::Repository", + { + "DomainName": config["DOMAIN_NAME"], + "DomainOwner": config["AWS_ACCOUNT"], + "RepositoryName": config["INTERNAL_SHARED_REPO"], + "PermissionsPolicyDocument": { + "Statement": [ + { + "Condition": { + "StringEquals": {"aws:PrincipalOrgId": [config["AWS_ORG_ID"]]} + }, + "Effect": "Allow", + "Sid": "ReadAccess", + }, + { + "Condition": { + "ArnLike": { + "aws:PrincipalArn": [ + "arn:aws:iam::459220104973:role/*/github/GHRunnerExecRole*", + "arn:aws:iam::529177531254:role/*/github/GHRunnerExecRole*", + ] + } + }, + "Effect": "Allow", + "Principal": { + "AWS": [ + "arn:aws:iam::459220104973:root", + "arn:aws:iam::529177531254:root", + ] + }, + "Sid": "WriteAccess", + }, + ] + }, + "Upstreams": [ + "external-crates-io", + "external-maven-central", + "external-maven-clojars", + "external-maven-commonsware", + "external-maven-googleandroid", + "external-maven-gradleplugins", + "external-npmjs", + "external-nuget-org", + "external-pypi", + "external-ruby-gems-org", + ], + }, + ) diff --git a/CodeArtifact/cdk/central_resources/tests/Makefile b/CodeArtifact/cdk/central_resources/tests/Makefile new file mode 100644 index 00000000..d0bd2e43 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/tests/Makefile @@ -0,0 +1,23 @@ +export ENV ?= dev + +ROOT_PATH := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))/../) +ENV_FILE := $(ROOT_PATH)/environment/$(ENV).yaml + +export AWS_REGION=$(shell grep AWS_REGION $(ENV_FILE) | awk -F ': ' '{print($$2)}') +export DOMAIN_NAME=$(shell grep DOMAIN_NAME $(ENV_FILE) | awk -F ': ' '{print($$2)}') +export DOMAIN_OWNER=$(shell grep AWS_ACCOUNT $(ENV_FILE) | awk -F ': ' '{gsub(/"/, "", $$2); print($$2)}') +export TARGET_REPO_NAME=$(shell grep INTERNAL_SHARED_REPO $(ENV_FILE) | awk -F ': ' '{print($$2)}') + +test-repo-exist: + aws codeartifact describe-repository --domain $(DOMAIN_NAME) --domain-owner $(DOMAIN_OWNER) --repository $(TARGET_REPO_NAME) --region $(AWS_REGION) + +test-generic: + cd generic && ./test_generic_pkg_publish_download.sh + +test-nuget: + cd nuget && ./test_nuget_method_2.sh + +test-python: + cd python && ./test_twine_publish_pip_download.sh + +pre-commit: test-repo-exist test-generic test-nuget test-python diff --git a/CodeArtifact/cdk/central_resources/tests/generic/package_dummy_targz.sh b/CodeArtifact/cdk/central_resources/tests/generic/package_dummy_targz.sh new file mode 100644 index 00000000..f076faa7 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/tests/generic/package_dummy_targz.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +echo "CheckPt: Building ${PKG_NAME}" + +cat < app.sh +echo "dummy-app-1.0.0" +EOT + +tar -czf ${PKG_NAME}.tar.gz app.sh + +rm app.sh diff --git a/CodeArtifact/cdk/central_resources/tests/generic/test_generic_pkg_publish_download.sh b/CodeArtifact/cdk/central_resources/tests/generic/test_generic_pkg_publish_download.sh new file mode 100644 index 00000000..047b8ce0 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/tests/generic/test_generic_pkg_publish_download.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -e + +echo "######################################################################" +echo "CheckPt: Install tools" + +pip3 install -i https://pypi.org/simple -U awscli + +echo "######################################################################" +echo "CheckPt: Package a dummy tar.gz package" + +export PKG_NAME="dummyasset$(date +%Y%m%d%H%M%S)" +./package_dummy_targz.sh + +PKG=${PKG_NAME}.tar.gz +PKG_NS=unittest + +echo "######################################################################" +echo "CheckPt: Test publish" + +export ASSET_SHA256=$(sha256sum ${PKG} | awk '{print $1;}') + +# Generic packages must have a namespace +aws codeartifact publish-package-version --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} \ + --format generic --namespace ${PKG_NS} --package ${PKG_NAME} \ + --package-version "1.0.0" \ + --asset-content ${PKG} --asset-name ${PKG} \ + --asset-sha256 $ASSET_SHA256 \ + --region ${AWS_REGION} + +echo "######################################################################" +echo "CheckPt: Test list-package-versions" + +aws codeartifact list-package-versions --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} \ + --format generic --namespace ${PKG_NS} --package ${PKG_NAME} \ + --region ${AWS_REGION} + +echo "######################################################################" +echo "CheckPt: Test download generic package assets ${PKG_NAME}" + +aws codeartifact get-package-version-asset --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} \ + --format generic --namespace ${PKG_NS} --package ${PKG_NAME} \ + --package-version "1.0.0" \ + --asset ${PKG} \ + ${PKG} \ + --region ${AWS_REGION} + +tar -xvf ${PKG} +rm ${PKG} app.sh + +echo "######################################################################" +echo "CheckPt: Test delete package" + +aws codeartifact delete-package --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} \ + --format generic --namespace ${PKG_NS} --package ${PKG_NAME} \ + --region ${AWS_REGION} diff --git a/CodeArtifact/cdk/central_resources/tests/nuget/create_dummy_consumer_csproj.sh b/CodeArtifact/cdk/central_resources/tests/nuget/create_dummy_consumer_csproj.sh new file mode 100644 index 00000000..c8977b33 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/tests/nuget/create_dummy_consumer_csproj.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +cat < Dummy.CodeArtifact.Consumer.csproj + + + netstandard2.0 + Dummy.CodeArtifact.Consumer + 1.0.0 + kyhau + Dummy package for CodeArtifact unit tests + + +EOT diff --git a/CodeArtifact/cdk/central_resources/tests/nuget/package_dummy_nupkg.sh b/CodeArtifact/cdk/central_resources/tests/nuget/package_dummy_nupkg.sh new file mode 100644 index 00000000..b0d3418c --- /dev/null +++ b/CodeArtifact/cdk/central_resources/tests/nuget/package_dummy_nupkg.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +echo "CheckPt: Building ${PKG_NAME}" + +cat < ${PKG_NAME}.csproj + + + netstandard2.0 + ${PKG_NAME} + 1.0.0 + kyhau + Dummy package for CodeArtifact unit tests + + +EOT + +dotnet pack ${PKG_NAME}.csproj + +rm -rf ${PKG_NAME}.csproj obj/ bin/Debug/netstandard2.0/ diff --git a/CodeArtifact/cdk/central_resources/tests/nuget/test_nuget_method_1.sh b/CodeArtifact/cdk/central_resources/tests/nuget/test_nuget_method_1.sh new file mode 100644 index 00000000..23854e52 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/tests/nuget/test_nuget_method_1.sh @@ -0,0 +1,71 @@ +#!/bin/bash +set -e +# See https://docs.aws.amazon.com/codeartifact/latest/ug/nuget-cli.html +# +# Method 1: Configure with the CodeArtifact NuGet Credential Provider +# This test script uses/requires dotnet. + +echo "######################################################################" +echo "CheckPt: Install tools" + +pip3 install -i https://pypi.org/simple -U awscli + +echo "######################################################################" +echo "CheckPt: Configure the nuget or dotnet CLI" + +# Backup NuGet.Config if exists +if [ -f ~/.nuget/NuGet/NuGet.Config ]; then + cp ~/.nuget/NuGet/NuGet.Config ~/.nuget/NuGet/NuGet.Config.bkup +fi + +dotnet tool install -g AWS.CodeArtifact.NuGet.CredentialProvider +dotnet codeartifact-creds install + +repositoryEndpoint=$(aws codeartifact get-repository-endpoint --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} --region ${AWS_REGION} --format nuget | jq -r ".repositoryEndpoint") + +dotnet nuget add source ${repositoryEndpoint}v3/index.json --name ${DOMAIN_NAME}/${TARGET_REPO_NAME} + +echo "######################################################################" +echo "CheckPt: Package a dummy nupkg" + +export PKG_NAME="Dummy.CodeArtifact.Test$(date +%Y%m%d%H%M%S)" +PKG=bin/Debug/${PKG_NAME}.1.0.0.nupkg +./package_dummy_nupkg.sh + +echo "######################################################################" +echo "CheckPt: Test publish" + +dotnet nuget push ${PKG} --source ${DOMAIN_NAME}/${TARGET_REPO_NAME} + +echo "######################################################################" +echo "CheckPt: Test list-package-versions" + +aws codeartifact list-package-versions --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} --format nuget --package ${PKG_NAME} --region ${AWS_REGION} + +echo "######################################################################" +echo "CheckPt: Test install ${PKG_NAME}" + +./create_dummy_consumer_csproj.sh + +dotnet add package ${PKG_NAME} --version "1.0.0" + +echo "######################################################################" +echo "CheckPt: Test delete package" + +aws codeartifact delete-package --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} --format nuget --package ${PKG_NAME} --region ${AWS_REGION} + +echo "######################################################################" +echo "CheckPt: Cleanup" + +# Restore NuGet.Config if exists +if [ -f ~/.nuget/NuGet/NuGet.Config.bkup ]; then + mv ~/.nuget/NuGet/NuGet.Config.bkup ~/.nuget/NuGet/NuGet.Config +fi + +rm -rf *.csproj bin/ obj/ + +dotnet codeartifact-creds uninstall --delete-configuration +dotnet tool uninstall -g AWS.CodeArtifact.NuGet.CredentialProvider diff --git a/CodeArtifact/cdk/central_resources/tests/nuget/test_nuget_method_2.sh b/CodeArtifact/cdk/central_resources/tests/nuget/test_nuget_method_2.sh new file mode 100644 index 00000000..de9f2d5b --- /dev/null +++ b/CodeArtifact/cdk/central_resources/tests/nuget/test_nuget_method_2.sh @@ -0,0 +1,63 @@ +#!/bin/bash +set -e +# See https://docs.aws.amazon.com/codeartifact/latest/ug/nuget-cli.html +# +# Method 2: Configure nuget or dotnet with the `codeartifact login` command +# This test script uses/requires dotnet. + +echo "######################################################################" +echo "CheckPt: Install tools" + +pip3 install -i https://pypi.org/simple -U awscli + +echo "######################################################################" +echo "CheckPt: Configure the nuget or dotnet CLI" + +# Backup NuGet.Config if exists +if [ -f ~/.nuget/NuGet/NuGet.Config ]; then + cp ~/.nuget/NuGet/NuGet.Config ~/.nuget/NuGet/NuGet.Config.bkup +fi + +aws codeartifact login --tool dotnet --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} --region ${AWS_REGION} + +echo "######################################################################" +echo "CheckPt: Package a dummy nupkg" + +export PKG_NAME="Dummy.CodeArtifact.Test$(date +%Y%m%d%H%M%S)" +PKG=bin/Debug/${PKG_NAME}.1.0.0.nupkg +./package_dummy_nupkg.sh + +echo "######################################################################" +echo "CheckPt: Test publish" + +dotnet nuget push ${PKG} --source ${DOMAIN_NAME}/${TARGET_REPO_NAME} + +echo "######################################################################" +echo "CheckPt: Test list-package-versions" + +aws codeartifact list-package-versions --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} --format nuget --package ${PKG_NAME} --region ${AWS_REGION} + +echo "######################################################################" +echo "CheckPt: Test install ${PKG_NAME}" + +./create_dummy_consumer_csproj.sh + +dotnet add package ${PKG_NAME} --version "1.0.0" + +echo "######################################################################" +echo "CheckPt: Test delete package" + +aws codeartifact delete-package --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} --format nuget --package ${PKG_NAME} --region ${AWS_REGION} + +echo "######################################################################" +echo "CheckPt: Cleanup" + +# Restore NuGet.Config if exists +if [ -f ~/.nuget/NuGet/NuGet.Config.bkup ]; then + mv ~/.nuget/NuGet/NuGet.Config.bkup ~/.nuget/NuGet/NuGet.Config +fi + +rm -rf *.csproj bin/ obj/ diff --git a/CodeArtifact/cdk/central_resources/tests/nuget/test_nuget_method_3.sh b/CodeArtifact/cdk/central_resources/tests/nuget/test_nuget_method_3.sh new file mode 100644 index 00000000..c802cba6 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/tests/nuget/test_nuget_method_3.sh @@ -0,0 +1,70 @@ +#!/bin/bash +set -e +# See https://docs.aws.amazon.com/codeartifact/latest/ug/nuget-cli.html +# +# Method 3: Configure nuget or dotnet without the login command +# Manually configure nuget or dotnet to connect to your CodeArtifact repository. +# This test script uses/requires dotnet. + +echo "######################################################################" +echo "CheckPt: Install tools" + +pip3 install -i https://pypi.org/simple -U awscli + +echo "######################################################################" +echo "CheckPt: Configure nuget or dotnet to connect to your CodeArtifact repository" + +# Backup NuGet.Config if exists +if [ -f ~/.nuget/NuGet/NuGet.Config ]; then + cp ~/.nuget/NuGet/NuGet.Config ~/.nuget/NuGet/NuGet.Config.bkup +fi + +repositoryEndpoint=$(aws codeartifact get-repository-endpoint --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} --region ${AWS_REGION} --format nuget | jq -r ".repositoryEndpoint") + +# Note for Linux and MacOS users: +# Because encryption is not supported on non-Windows platforms, you must add the --store-password-in-clear-text flag to the following command. +dotnet nuget add source ${repositoryEndpoint}v3/index.json \ + --name ${DOMAIN_NAME}/${TARGET_REPO_NAME} --username aws --store-password-in-clear-text \ + --password $(aws codeartifact get-authorization-token --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} | jq -r ".authorizationToken") + +echo "######################################################################" +echo "CheckPt: Package a dummy nupkg" + +export PKG_NAME="Dummy.CodeArtifact.Test$(date +%Y%m%d%H%M%S)" +PKG=bin/Debug/${PKG_NAME}.1.0.0.nupkg +./package_dummy_nupkg.sh + +echo "######################################################################" +echo "CheckPt: Test publish" + +dotnet nuget push ${PKG} --source ${DOMAIN_NAME}/${TARGET_REPO_NAME} + +echo "######################################################################" +echo "CheckPt: Test list-package-versions" + +aws codeartifact list-package-versions --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} --format nuget --package ${PKG_NAME} --region ${AWS_REGION} + +echo "######################################################################" +echo "CheckPt: Test install ${PKG_NAME}" + +./create_dummy_consumer_csproj.sh + +dotnet add package ${PKG_NAME} --version "1.0.0" + +echo "######################################################################" +echo "CheckPt: Test delete package" + +aws codeartifact delete-package --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} --format nuget --package ${PKG_NAME} --region ${AWS_REGION} + +echo "######################################################################" +echo "CheckPt: Cleanup" + +# Restore NuGet.Config if exists +if [ -f ~/.nuget/NuGet/NuGet.Config.bkup ]; then + mv ~/.nuget/NuGet/NuGet.Config.bkup ~/.nuget/NuGet/NuGet.Config +fi + +rm -rf *.csproj bin/ obj/ diff --git a/CodeArtifact/cdk/central_resources/tests/python/package_dummy_whl.sh b/CodeArtifact/cdk/central_resources/tests/python/package_dummy_whl.sh new file mode 100644 index 00000000..f7791660 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/tests/python/package_dummy_whl.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -e + +echo "CheckPt: Building ${PKG_NAME}" + +mkdir -p app/ + +cat < app/__init__.py +VERSION = "1.0.0" +EOT + +cat < app/test.py +from . import VERSION +def print_version(): + print(VERSION) +EOT + +cat < setup-app.py +from setuptools import setup +setup( + name='${PKG_NAME}', + version='1.0.0', + packages=['.app'], +) +EOT + +pip3 install wheel setuptools + +python3 setup-app.py bdist_wheel --universal --bdist-dir ~/temp/bdistwheel + +rm -rf app/ setup-app.py *.egg-info build dist/app dist/*.egg-info dist/*.dist-info diff --git a/CodeArtifact/cdk/central_resources/tests/python/test_twine_publish_pip_download.sh b/CodeArtifact/cdk/central_resources/tests/python/test_twine_publish_pip_download.sh new file mode 100644 index 00000000..9269e914 --- /dev/null +++ b/CodeArtifact/cdk/central_resources/tests/python/test_twine_publish_pip_download.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -e + +echo "######################################################################" +echo "CheckPt: Install tools" + +pip3 install -i https://pypi.org/simple -U awscli twine + +echo "######################################################################" +echo "CheckPt: Package a dummy wheel" + +export PKG_NAME="dummypythonapp$(date +%Y%m%d%H%M%S)" +PKG=dist/${PKG_NAME}-1.0.0-py2.py3-none-any.whl +./package_dummy_whl.sh + +echo "######################################################################" +echo "CheckPt: Test publish" + +aws codeartifact login --tool twine --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} --region ${AWS_REGION} + +twine upload --repository codeartifact ${PKG} --verbose + +rm ~/.pypirc +rm -rf dist/ + +echo "######################################################################" +echo "CheckPt: Test list-package-versions" + +aws codeartifact list-package-versions --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} --format pypi --package ${PKG_NAME} --region ${AWS_REGION} + +echo "######################################################################" +echo "CheckPt: Test pip3 install ${PKG_NAME}" + +aws codeartifact login --tool pip --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} --region ${AWS_REGION} + +pip3 install ${PKG_NAME} + +rm ~/.config/pip/pip.conf + +echo "######################################################################" +echo "CheckPt: Test delete package" + +aws codeartifact delete-package --domain ${DOMAIN_NAME} --domain-owner ${DOMAIN_OWNER} \ + --repository ${TARGET_REPO_NAME} --format pypi --package ${PKG_NAME} --region ${AWS_REGION} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..aa4949aa --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.black] +line-length = 100 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..be626e63 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,6 @@ +[flake8] +extend-ignore = + E501, + F541, + W503, + W605