From 3265cfd8dc019124854b2c65890fc48c649b4e80 Mon Sep 17 00:00:00 2001 From: George Hudson Date: Tue, 1 Aug 2023 07:04:11 -0600 Subject: [PATCH] moved erd and infrastructure workflows into nested jobs in deploy (#2632) * moved erd and infrastructure workflows into nested jobs in deploy * forgot to save commands file * keeping make_erd job and commands definitions in separate folder --------- Co-authored-by: George Hudson Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com> --- .circleci/deployment/commands.yml | 115 +++++++++++++++++++++++++ .circleci/deployment/jobs.yml | 91 +++++++++++++++++++ .circleci/deployment/workflows.yml | 70 +++++++++++++-- .circleci/generate_config.sh | 4 - .circleci/infrastructure/commands.yml | 115 ------------------------- .circleci/infrastructure/jobs.yml | 91 ------------------- .circleci/infrastructure/workflows.yml | 42 --------- .circleci/util/workflows.yml | 8 -- 8 files changed, 267 insertions(+), 269 deletions(-) delete mode 100644 .circleci/infrastructure/commands.yml delete mode 100644 .circleci/infrastructure/jobs.yml delete mode 100644 .circleci/infrastructure/workflows.yml delete mode 100644 .circleci/util/workflows.yml diff --git a/.circleci/deployment/commands.yml b/.circleci/deployment/commands.yml index b259fb57f1..54bd7d1084 100644 --- a/.circleci/deployment/commands.yml +++ b/.circleci/deployment/commands.yml @@ -149,3 +149,118 @@ else echo "export DEPLOY_STRATEGY=rolling" >> $BASH_ENV fi + + deploy-infrastructure: + parameters: + tf-path: + type: string + default: ./terraform/dev + cf-password: + type: env_var_name + default: CF_PASSWORD_DEV + cf-username: + type: env_var_name + default: CF_USERNAME_DEV + cf-space: + type: string + default: tanf-dev + cf-org: + type: env_var_name + default: CF_ORG + cf-app: + type: string + default: CF_APP + steps: + - checkout + - run: + name: Install dependencies + command: | + apk update + apk add jq + apk add curl + # TODO: Add Signature check + curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v7&source=github" | tar -zx + mv cf7 /usr/local/bin/cf + - login-cloud-dot-gov: + cf-password: <> + cf-username: <> + cf-space: <> + - run: + name: Export S3 Credentials for TFState + command: | + S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2) + { + echo "access_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .access_key_id)\"" + echo "secret_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .secret_access_key)\"" + echo "region = \"$(echo "${S3_CREDENTIALS}" | jq -r '.region')\"" + echo "bucket = \"$(echo "${S3_CREDENTIALS}" | jq -r '.bucket')\"" + } >> ./backend_config.tfvars + - run: + name: Prepare Terraform Variables + command: | + S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2) + { + echo "cf_password = \"$<>\"" + echo "cf_user = \"$<>\"" + echo "cf_space_name = \"<>\"" + echo "cf_app_name = \"<>\"" + } >> ./variables.tfvars + - terraform/init: + path: <> + backend_config_file: ./backend_config.tfvars + - terraform/validate: + path: <> + - terraform/fmt: + path: <> + - terraform/plan: + path: <> + var_file: ./variables.tfvars + - terraform/apply: + path: <> + var_file: ./variables.tfvars + + enable-s3-versioning: + parameters: + target_env: + type: string + cf-password: + type: env_var_name + cf-username: + type: env_var_name + cf-space: + type: string + steps: + - checkout + - run: + name: Install dependencies + command: | + sudo apt update + sudo apt install jq + sudo apt install curl + # TODO: Add Signature check + curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v7&source=github" | tar -zx + sudo mv cf7 /usr/local/bin/cf + sudo chmod +x /usr/local/bin/cf + - login-cloud-dot-gov: + cf-password: <> + cf-username: <> + cf-space: <> + - run: + name: Ensure s3 versioning key exists + # use cf to check if a service key exists for tdp-datafiles-dev(or prod) + # exit 0 if it does, create it if it doesn't + command: | + chmod +x ./scripts/create_s3_versioning_key.sh + ./scripts/create_s3_versioning_key.sh tdp-datafiles-<> s3-versioning-key + - run: + name: Set up aws credentials + command: | + S3_CREDENTIALS=$(cf service-key tdp-datafiles-<> s3-versioning-key | tail -n +2) + aws configure set aws_access_key_id $(echo "${S3_CREDENTIALS}" | jq -r .access_key_id) + aws configure set aws_secret_access_key $(echo "${S3_CREDENTIALS}" | jq -r .secret_access_key) + aws configure set default.region $(echo "${S3_CREDENTIALS}" | jq -r '.region') + - run: + name: Enable S3 Versioning + command: | + S3_BUCKET_ID=cg-$(cf service tdp-datafiles-<> --guid) + aws s3api put-bucket-versioning --bucket ${S3_BUCKET_ID} --versioning-configuration Status=Enabled diff --git a/.circleci/deployment/jobs.yml b/.circleci/deployment/jobs.yml index a24baef461..542f331361 100644 --- a/.circleci/deployment/jobs.yml +++ b/.circleci/deployment/jobs.yml @@ -32,6 +32,97 @@ cf-space: tanf-staging cf-username: CF_USERNAME_STAGING + deploy-infrastructure-dev: + executor: terraform/default + working_directory: ~/tdp-deploy + parameters: + target_env: + type: string + steps: + - deploy-infrastructure: + cf-app: << parameters.target_env >> + + deploy-infrastructure-staging: + executor: terraform/default + working_directory: ~/tdp-deploy + steps: + - deploy-infrastructure: + cf-password: CF_PASSWORD_STAGING + cf-username: CF_USERNAME_STAGING + cf-space: tanf-staging + tf-path: ./terraform/staging + + deploy-infrastructure-develop: + executor: terraform/default + working_directory: ~/tdp-deploy + steps: + - deploy-infrastructure: + cf-password: CF_PASSWORD_STAGING + cf-username: CF_USERNAME_STAGING + cf-space: tanf-staging + tf-path: ./terraform/staging + + deploy-infrastructure-production: + executor: terraform/default + working_directory: ~/tdp-deploy + steps: + - deploy-infrastructure: + cf-password: CF_PASSWORD_PROD + cf-username: CF_USERNAME_PROD + cf-space: tanf-prod + tf-path: ./terraform/production + + deploy-project-updates-site: + parameters: + cf-org: + default: CF_ORG + type: env_var_name + cf-space: + default: tanf-dev + type: string + cf-password: + type: env_var_name + default: CF_PASSWORD_DEV + cf-username: + type: env_var_name + default: CF_USERNAME_DEV + executor: docker-executor + working_directory: ~/tdp-deploy + steps: + - checkout + - sudo-check + - cf-check + - login-cloud-dot-gov: + cf-password: <> + cf-org: <> + cf-space: <> + cf-username: <> + - run: + name: Deploy TDP Project Updates Site + command: ./scripts/deploy-tdp-product-update-site.sh rolling tdp-project-updates + + enable-versioning: + executor: machine-executor + parameters: + target_env: + type: string + default: dev + cf-password: + type: env_var_name + default: CF_PASSWORD_DEV + cf-username: + type: env_var_name + default: CF_USERNAME_DEV + cf-space: + type: string + default: tanf-dev + steps: + - enable-s3-versioning: + target_env: <> + cf-password: <> + cf-username: <> + cf-space: <> + test-deployment-e2e: executor: machine-executor working_directory: ~/tdp-apps diff --git a/.circleci/deployment/workflows.yml b/.circleci/deployment/workflows.yml index 91e5ea56a9..6ecaae41fe 100644 --- a/.circleci/deployment/workflows.yml +++ b/.circleci/deployment/workflows.yml @@ -5,6 +5,15 @@ jobs: - deploy-infrastructure-dev: target_env: << pipeline.parameters.target_env >> + - enable-versioning: + requires: + - deploy-infrastructure-dev + filters: + branches: + ignore: + - develop + - main + - master - deploy-dev: target_env: << pipeline.parameters.target_env >> requires: @@ -21,19 +30,18 @@ branches: only: - develop - - deploy-infrastructure-staging: + - deploy-infrastructure-develop: filters: branches: only: - - main - - deploy-staging: + - develop + - enable-versioning: requires: - - deploy-infrastructure-staging - filters: - branches: - only: - - main - - deploy-infrastructure-develop: + - deploy-infrastructure-develop + target_env: develop + cf-password: CF_PASSWORD_STAGING + cf-username: CF_USERNAME_STAGING + cf-space: tanf-staging filters: branches: only: @@ -52,6 +60,34 @@ branches: only: - develop + - make_erd: # from ../util folder + filters: + branches: + only: + - develop + - deploy-infrastructure-staging: + filters: + branches: + only: + - main + - enable-versioning: + requires: + - deploy-infrastructure-staging + target_env: staging + cf-password: CF_PASSWORD_STAGING + cf-username: CF_USERNAME_STAGING + cf-space: tanf-staging + filters: + branches: + only: + - main + - deploy-staging: + requires: + - deploy-infrastructure-staging + filters: + branches: + only: + - main production-deployment: unless: @@ -64,6 +100,17 @@ branches: only: - master + - enable-versioning: + requires: + - deploy-infrastructure-production + target_env: prod + cf-password: CF_PASSWORD_PROD + cf-username: CF_USERNAME_PROD + cf-space: tanf-prod + filters: + branches: + only: + - master - deploy-production: requires: - deploy-infrastructure-production @@ -71,4 +118,9 @@ branches: only: - master + - make_erd: # from ../util folder + filters: + branches: + only: + - master diff --git a/.circleci/generate_config.sh b/.circleci/generate_config.sh index 74cab13120..0349eb3520 100644 --- a/.circleci/generate_config.sh +++ b/.circleci/generate_config.sh @@ -5,7 +5,6 @@ cat base_config.yml > generated_config.yml # Commands echo 'commands:' >> generated_config.yml cat build-and-test/commands.yml >> generated_config.yml -cat infrastructure/commands.yml >> generated_config.yml cat deployment/commands.yml >> generated_config.yml cat owasp/commands.yml >> generated_config.yml cat util/commands.yml >> generated_config.yml @@ -13,7 +12,6 @@ cat util/commands.yml >> generated_config.yml # Jobs echo 'jobs:' >> generated_config.yml cat build-and-test/jobs.yml >> generated_config.yml -cat infrastructure/jobs.yml >> generated_config.yml cat deployment/jobs.yml >> generated_config.yml cat owasp/jobs.yml >> generated_config.yml cat util/jobs.yml >> generated_config.yml @@ -21,9 +19,7 @@ cat util/jobs.yml >> generated_config.yml # Workflows echo 'workflows:' >> generated_config.yml cat build-and-test/workflows.yml >> generated_config.yml -cat infrastructure/workflows.yml >> generated_config.yml cat deployment/workflows.yml >> generated_config.yml cat owasp/workflows.yml >> generated_config.yml -cat util/workflows.yml >> generated_config.yml cat generated_config.yml \ No newline at end of file diff --git a/.circleci/infrastructure/commands.yml b/.circleci/infrastructure/commands.yml deleted file mode 100644 index b6bcd40ef5..0000000000 --- a/.circleci/infrastructure/commands.yml +++ /dev/null @@ -1,115 +0,0 @@ -# commands: - deploy-infrastructure: - parameters: - tf-path: - type: string - default: ./terraform/dev - cf-password: - type: env_var_name - default: CF_PASSWORD_DEV - cf-username: - type: env_var_name - default: CF_USERNAME_DEV - cf-space: - type: string - default: tanf-dev - cf-org: - type: env_var_name - default: CF_ORG - cf-app: - type: string - default: CF_APP - steps: - - checkout - - run: - name: Install dependencies - command: | - apk update - apk add jq - apk add curl - # TODO: Add Signature check - curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v7&source=github" | tar -zx - mv cf7 /usr/local/bin/cf - - login-cloud-dot-gov: - cf-password: <> - cf-username: <> - cf-space: <> - - run: - name: Export S3 Credentials for TFState - command: | - S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2) - { - echo "access_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .access_key_id)\"" - echo "secret_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .secret_access_key)\"" - echo "region = \"$(echo "${S3_CREDENTIALS}" | jq -r '.region')\"" - echo "bucket = \"$(echo "${S3_CREDENTIALS}" | jq -r '.bucket')\"" - } >> ./backend_config.tfvars - - run: - name: Prepare Terraform Variables - command: | - S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2) - { - echo "cf_password = \"$<>\"" - echo "cf_user = \"$<>\"" - echo "cf_space_name = \"<>\"" - echo "cf_app_name = \"<>\"" - } >> ./variables.tfvars - - terraform/init: - path: <> - backend_config_file: ./backend_config.tfvars - - terraform/validate: - path: <> - - terraform/fmt: - path: <> - - terraform/plan: - path: <> - var_file: ./variables.tfvars - - terraform/apply: - path: <> - var_file: ./variables.tfvars - - enable-s3-versioning: - parameters: - target_env: - type: string - cf-password: - type: env_var_name - cf-username: - type: env_var_name - cf-space: - type: string - steps: - - checkout - - run: - name: Install dependencies - command: | - sudo apt update - sudo apt install jq - sudo apt install curl - # TODO: Add Signature check - curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v7&source=github" | tar -zx - sudo mv cf7 /usr/local/bin/cf - sudo chmod +x /usr/local/bin/cf - - login-cloud-dot-gov: - cf-password: <> - cf-username: <> - cf-space: <> - - run: - name: Ensure s3 versioning key exists - # use cf to check if a service key exists for tdp-datafiles-dev(or prod) - # exit 0 if it does, create it if it doesn't - command: | - chmod +x ./scripts/create_s3_versioning_key.sh - ./scripts/create_s3_versioning_key.sh tdp-datafiles-<> s3-versioning-key - - run: - name: Set up aws credentials - command: | - S3_CREDENTIALS=$(cf service-key tdp-datafiles-<> s3-versioning-key | tail -n +2) - aws configure set aws_access_key_id $(echo "${S3_CREDENTIALS}" | jq -r .access_key_id) - aws configure set aws_secret_access_key $(echo "${S3_CREDENTIALS}" | jq -r .secret_access_key) - aws configure set default.region $(echo "${S3_CREDENTIALS}" | jq -r '.region') - - run: - name: Enable S3 Versioning - command: | - S3_BUCKET_ID=cg-$(cf service tdp-datafiles-<> --guid) - aws s3api put-bucket-versioning --bucket ${S3_BUCKET_ID} --versioning-configuration Status=Enabled diff --git a/.circleci/infrastructure/jobs.yml b/.circleci/infrastructure/jobs.yml deleted file mode 100644 index f3385d9a8f..0000000000 --- a/.circleci/infrastructure/jobs.yml +++ /dev/null @@ -1,91 +0,0 @@ -# jobs: - deploy-infrastructure-dev: - executor: terraform/default - working_directory: ~/tdp-deploy - parameters: - target_env: - type: string - steps: - - deploy-infrastructure: - cf-app: << parameters.target_env >> - - deploy-infrastructure-staging: - executor: terraform/default - working_directory: ~/tdp-deploy - steps: - - deploy-infrastructure: - cf-password: CF_PASSWORD_STAGING - cf-username: CF_USERNAME_STAGING - cf-space: tanf-staging - tf-path: ./terraform/staging - - deploy-infrastructure-develop: - executor: terraform/default - working_directory: ~/tdp-deploy - steps: - - deploy-infrastructure: - cf-password: CF_PASSWORD_STAGING - cf-username: CF_USERNAME_STAGING - cf-space: tanf-staging - tf-path: ./terraform/staging - - deploy-infrastructure-production: - executor: terraform/default - working_directory: ~/tdp-deploy - steps: - - deploy-infrastructure: - cf-password: CF_PASSWORD_PROD - cf-username: CF_USERNAME_PROD - cf-space: tanf-prod - tf-path: ./terraform/production - - deploy-project-updates-site: - parameters: - cf-org: - default: CF_ORG - type: env_var_name - cf-space: - default: tanf-dev - type: string - cf-password: - type: env_var_name - default: CF_PASSWORD_DEV - cf-username: - type: env_var_name - default: CF_USERNAME_DEV - executor: docker-executor - working_directory: ~/tdp-deploy - steps: - - checkout - - sudo-check - - cf-check - - login-cloud-dot-gov: - cf-password: <> - cf-org: <> - cf-space: <> - cf-username: <> - - run: - name: Deploy TDP Project Updates Site - command: ./scripts/deploy-tdp-product-update-site.sh rolling tdp-project-updates - - enable-versioning: - executor: machine-executor - parameters: - target_env: - type: string - default: dev - cf-password: - type: env_var_name - default: CF_PASSWORD_DEV - cf-username: - type: env_var_name - default: CF_USERNAME_DEV - cf-space: - type: string - default: tanf-dev - steps: - - enable-s3-versioning: - target_env: <> - cf-password: <> - cf-username: <> - cf-space: <> diff --git a/.circleci/infrastructure/workflows.yml b/.circleci/infrastructure/workflows.yml deleted file mode 100644 index 9cb85e8ed8..0000000000 --- a/.circleci/infrastructure/workflows.yml +++ /dev/null @@ -1,42 +0,0 @@ -#workflows: - enable-versioning-for-s3-buckets: - when: - or: - - << pipeline.parameters.deploy_infrastructure >> - - equal: [ 'main', << pipeline.git.branch >> ] - - equal: [ 'master', << pipeline.git.branch >> ] - jobs: - - enable-versioning: - filters: - branches: - ignore: - - develop - - main - - master - - enable-versioning: - target_env: staging - cf-password: CF_PASSWORD_STAGING - cf-username: CF_USERNAME_STAGING - cf-space: tanf-staging - filters: - branches: - only: - - main - - enable-versioning: - target_env: prod - cf-password: CF_PASSWORD_PROD - cf-username: CF_USERNAME_PROD - cf-space: tanf-prod - filters: - branches: - only: - - master - - enable-versioning: - target_env: develop - cf-password: CF_PASSWORD_STAGING - cf-username: CF_USERNAME_STAGING - cf-space: tanf-staging - filters: - branches: - only: - - develop diff --git a/.circleci/util/workflows.yml b/.circleci/util/workflows.yml deleted file mode 100644 index 116e30bf36..0000000000 --- a/.circleci/util/workflows.yml +++ /dev/null @@ -1,8 +0,0 @@ -# workflows: - erd: - when: - or: - - equal: [ 'develop', << pipeline.git.branch >> ] - - equal: [ 'master', << pipeline.git.branch >> ] - jobs: - - make_erd