moved erd and infrastructure workflows into nested jobs in deploy

.circleci/deployment/commands.yml

@@ -149,3 +149,171 @@
             else
               echo "export DEPLOY_STRATEGY=rolling" >> $BASH_ENV
             fi
+
+  docker-compose-check:
+    steps:
+      - run:
+          name: Ensure docker-compose exists, otherwise install it.
+          command: ./scripts/docker-compose-check.sh
+
+  docker-compose-up-backend:
+    steps:
+      - run:
+          name: Build and spin-up Django API service
+          command: cd tdrs-backend; docker network create external-net; docker-compose up -d --build
+
+  cf-check:
+    steps:
+      - run:
+          name: Ensure cf cli is installed, otherwise install it.
+          command: sudo ./scripts/cf-check.sh
+
+  # This allows us to use the orb stanza for node/install within other commands
+  # NOTE: This doesn't work correctly on machine executors
+  install-nodejs: node/install
+
+  sudo-check:
+    steps:
+      - run:
+          name: Ensure sudo is installed, otherwise install it.
+          command: ./scripts/sudo-check.sh
+
+  login-cloud-dot-gov:
+    description: Authenticates with Cloud.gov and sets org and space targets
+    parameters:
+      cf-password:
+        type: env_var_name
+        default: CF_PASSWORD_DEV
+      cf-org:
+        type: env_var_name
+        default: CF_ORG
+      cf-space:
+        type: string
+        default: tanf-dev
+      cf-username:
+        type: env_var_name
+        default: CF_USERNAME_DEV
+    steps:
+      - run:
+          name: Login to Cloud.gov and set application targets
+          command: |
+            cf login -a https://api.fr.cloud.gov \
+              -u ${<<parameters.cf-username>>} \
+              -p ${<<parameters.cf-password>>} \
+              -o ${<<parameters.cf-org>>} \
+              -s <<parameters.cf-space>>
+
+  deploy-infrastructure:
+    parameters:
+      tf-path:
+        type: string
+        default: ./terraform/dev
+      cf-password:
+        type: env_var_name
+        default: CF_PASSWORD_DEV
+      cf-username:
+        type: env_var_name
+        default: CF_USERNAME_DEV
+      cf-space:
+        type: string
+        default: tanf-dev
+      cf-org:
+        type: env_var_name
+        default: CF_ORG
+      cf-app:
+        type: string
+        default: CF_APP
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            apk update
+            apk add jq
+            apk add curl
+            # TODO: Add Signature check
+            curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v7&source=github" | tar -zx
+            mv cf7 /usr/local/bin/cf
+      - login-cloud-dot-gov:
+          cf-password: <<parameters.cf-password>>
+          cf-username: <<parameters.cf-username>>
+          cf-space: <<parameters.cf-space>>
+      - run:
+          name: Export S3 Credentials for TFState
+          command: |
+            S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2)
+            {
+              echo "access_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .access_key_id)\""
+              echo "secret_key = \"$(echo "${S3_CREDENTIALS}" | jq -r .secret_access_key)\""
+              echo "region = \"$(echo "${S3_CREDENTIALS}" | jq -r '.region')\""
+              echo "bucket = \"$(echo "${S3_CREDENTIALS}" | jq -r '.bucket')\""
+            } >> ./backend_config.tfvars
+      - run:
+          name: Prepare Terraform Variables
+          command: |
+            S3_CREDENTIALS=$(cf service-key tdp-tf-states tdp-tf-key | tail -n +2)
+            {
+              echo "cf_password = \"$<<parameters.cf-password>>\""
+              echo "cf_user = \"$<<parameters.cf-username>>\""
+              echo "cf_space_name = \"<<parameters.cf-space>>\""
+              echo "cf_app_name = \"<<parameters.cf-app>>\""
+            } >> ./variables.tfvars
+      - terraform/init:
+          path: <<parameters.tf-path>>
+          backend_config_file: ./backend_config.tfvars
+      - terraform/validate:
+          path: <<parameters.tf-path>>
+      - terraform/fmt:
+          path: <<parameters.tf-path>>
+      - terraform/plan:
+          path: <<parameters.tf-path>>
+          var_file: ./variables.tfvars
+      - terraform/apply:
+          path: <<parameters.tf-path>>
+          var_file: ./variables.tfvars
+
+  enable-s3-versioning:
+    parameters:
+      target_env:
+        type: string
+      cf-password:
+        type: env_var_name
+      cf-username:
+        type: env_var_name
+      cf-space:
+        type: string
+    steps:
+      - checkout
+      - run:
+          name: Install dependencies
+          command: |
+            sudo apt update
+            sudo apt install jq
+            sudo apt install curl
+            # TODO: Add Signature check
+            curl -L "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v7&source=github" | tar -zx
+            sudo mv cf7 /usr/local/bin/cf
+            sudo chmod +x /usr/local/bin/cf
+      - login-cloud-dot-gov:
+          cf-password: <<parameters.cf-password>>
+          cf-username: <<parameters.cf-username>>
+          cf-space: <<parameters.cf-space>>
+      - run:
+          name: Ensure s3 versioning key exists
+          # use cf to check if a service key exists for tdp-datafiles-dev(or prod) 
+          # exit 0 if it does, create it if it doesn't
+          command: |
+            chmod +x ./scripts/create_s3_versioning_key.sh
+            ./scripts/create_s3_versioning_key.sh tdp-datafiles-<<parameters.target_env>> s3-versioning-key
+      - run:
+          name: Set up aws credentials
+          command: |
+            S3_CREDENTIALS=$(cf service-key tdp-datafiles-<<parameters.target_env>> s3-versioning-key | tail -n +2)
+            aws configure set aws_access_key_id $(echo "${S3_CREDENTIALS}" | jq -r .access_key_id)
+            aws configure set aws_secret_access_key $(echo "${S3_CREDENTIALS}" | jq -r .secret_access_key)
+            aws configure set default.region $(echo "${S3_CREDENTIALS}" | jq -r '.region')
+      - run:
+          name: Enable S3 Versioning
+          command: |
+            S3_BUCKET_ID=cg-$(cf service tdp-datafiles-<<parameters.target_env>> --guid)
+            aws s3api put-bucket-versioning --bucket ${S3_BUCKET_ID} --versioning-configuration Status=Enabled

.circleci/deployment/jobs.yml

@@ -32,6 +32,116 @@
           cf-space: tanf-staging
           cf-username: CF_USERNAME_STAGING
 
+  deploy-infrastructure-dev:
+    executor: terraform/default
+    working_directory: ~/tdp-deploy
+    parameters:
+      target_env:
+        type: string
+    steps:
+      - deploy-infrastructure:
+          cf-app: << parameters.target_env >>
+
+  deploy-infrastructure-staging:
+    executor: terraform/default
+    working_directory: ~/tdp-deploy
+    steps:
+      - deploy-infrastructure:
+          cf-password: CF_PASSWORD_STAGING
+          cf-username: CF_USERNAME_STAGING
+          cf-space: tanf-staging
+          tf-path: ./terraform/staging
+
+  deploy-infrastructure-develop:
+    executor: terraform/default
+    working_directory: ~/tdp-deploy
+    steps:
+      - deploy-infrastructure:
+          cf-password: CF_PASSWORD_STAGING
+          cf-username: CF_USERNAME_STAGING
+          cf-space: tanf-staging
+          tf-path: ./terraform/staging
+
+  deploy-infrastructure-production:
+    executor: terraform/default
+    working_directory: ~/tdp-deploy
+    steps:
+      - deploy-infrastructure:
+          cf-password: CF_PASSWORD_PROD
+          cf-username: CF_USERNAME_PROD
+          cf-space: tanf-prod
+          tf-path: ./terraform/production
+
+  deploy-project-updates-site:
+    parameters:
+      cf-org:
+        default: CF_ORG
+        type: env_var_name
+      cf-space:
+        default: tanf-dev
+        type: string
+      cf-password:
+        type: env_var_name
+        default: CF_PASSWORD_DEV
+      cf-username:
+        type: env_var_name
+        default: CF_USERNAME_DEV
+    executor: docker-executor
+    working_directory: ~/tdp-deploy
+    steps:
+      - checkout
+      - sudo-check
+      - cf-check
+      - login-cloud-dot-gov:
+          cf-password: <<parameters.cf-password>>
+          cf-org: <<parameters.cf-org>>
+          cf-space: <<parameters.cf-space>>
+          cf-username: <<parameters.cf-username>>
+      - run:
+          name: Deploy TDP Project Updates Site
+          command: ./scripts/deploy-tdp-product-update-site.sh rolling tdp-project-updates
+
+  enable-versioning:
+    executor: machine-executor
+    parameters:
+      target_env:
+        type: string
+        default: dev
+      cf-password:
+        type: env_var_name
+        default: CF_PASSWORD_DEV
+      cf-username:
+        type: env_var_name
+        default: CF_USERNAME_DEV
+      cf-space:
+        type: string
+        default: tanf-dev
+    steps:
+      - enable-s3-versioning:
+          target_env: <<parameters.target_env>>
+          cf-password: <<parameters.cf-password>>
+          cf-username: <<parameters.cf-username>>
+          cf-space: <<parameters.cf-space>>
+
+
+  make_erd:
+    executor: machine-executor
+    working_directory: ~/tdp_apps
+    steps:
+      - checkout
+      - docker-compose-check
+      - run:
+          name: Run graph_models
+          command: |
+            cd tdrs-backend
+            if [ $(docker network inspect external-net 2>&1 | grep -c Scope) == 0 ]; then 
+            docker network create external-net
+            fi
+            docker-compose run --rm web bash -c \
+            "./manage.py graph_models -a -g -o tdp_erd.png"
+      - store_artifacts:
+          path: tdrs-backend/tdp_erd.png
+
   test-deployment-e2e:
     executor: machine-executor
     working_directory: ~/tdp-apps

.circleci/deployment/workflows.yml

@@ -5,6 +5,15 @@
     jobs:
       - deploy-infrastructure-dev:
           target_env: << pipeline.parameters.target_env >>
+      - enable-versioning:
+          requires:
+            - deploy-infrastructure-dev
+          filters:
+            branches:
+              ignore:
+                - develop
+                - main
+                - master
       - deploy-dev:
           target_env: << pipeline.parameters.target_env >>
           requires:
@@ -21,19 +30,18 @@
             branches:
               only:
                 - develop
-      - deploy-infrastructure-staging:
+      - deploy-infrastructure-develop:
           filters:
             branches:
               only:
-                - main
-      - deploy-staging:
+                - develop
+      - enable-versioning:
           requires:
-            - deploy-infrastructure-staging
-          filters:
-            branches:
-              only:
-                - main
-      - deploy-infrastructure-develop:
+            - deploy-infrastructure-develop
+          target_env: develop
+          cf-password: CF_PASSWORD_STAGING
+          cf-username: CF_USERNAME_STAGING
+          cf-space: tanf-staging
           filters:
             branches:
               only:
@@ -52,6 +60,34 @@
             branches:
               only:
                 - develop
+      - make_erd:
+          filters:
+            branches:
+              only:
+                - develop
+      - deploy-infrastructure-staging:
+          filters:
+            branches:
+              only:
+                - main
+      - enable-versioning:
+          requires:
+            - deploy-infrastructure-staging
+          target_env: staging
+          cf-password: CF_PASSWORD_STAGING
+          cf-username: CF_USERNAME_STAGING
+          cf-space: tanf-staging
+          filters:
+            branches:
+              only:
+                - main
+      - deploy-staging:
+          requires:
+            - deploy-infrastructure-staging
+          filters:
+            branches:
+              only:
+                - main
 
   production-deployment:
     unless:
@@ -64,11 +100,27 @@
             branches:
               only:
                 - master
+      - enable-versioning:
+          requires:
+            - deploy-infrastructure-production
+          target_env: prod
+          cf-password: CF_PASSWORD_PROD
+          cf-username: CF_USERNAME_PROD
+          cf-space: tanf-prod
+          filters:
+            branches:
+              only:
+                - master
       - deploy-production:
           requires:
             - deploy-infrastructure-production
           filters:
             branches:
               only:
                 - master
+      - make_erd:
+          filters:
+            branches:
+              only:
+                - master