pipeline filtering (#2538)

* pipeline changes that filter based on paths and branches. circle ci tracks specified branches in order to keep current functionality on HHS side. * updated syntax to be in line with build-all.yml * removed comma * WIP build flow docs * added Architecture Decision Record for the change to pipeline workflows * corrected file type of doc to .md --------- Co-authored-by: George Hudson <ghudson@teamraft.com> Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com>
HHS · Jun 27, 2023 · ab09660 · ab09660
1 parent f7dcaf6
commit ab09660
Show file tree

Hide file tree

Showing 13 changed files with 414 additions and 14 deletions.
diff --git a/.circleci/README.md b/.circleci/README.md
@@ -14,10 +14,10 @@ This script will generate a complete config for building, testing, and deploying
 ### Directory structure
 
 #### build-and-test
-Contains workflows, jobs, and commands for building and testing the application.
+Contains workflows, jobs, and commands for building and testing the application. For all development side builds, these are now triggered by GitHub Actions that serve as a filter so only the code that's changed is tested. See [build-all](../.github/workflows/build-all.yml), [build-backend](../.github/workflows/build-backend.yml), and [build-frontend](../.github/workflows/build-frontend.yml)
 
 #### infrastructure
-Contains workflows, jobs, and commands for setting up the infrastructure on Cloud gov.
+Contains workflows, jobs, and commands for setting up the infrastructure on Cloud gov. For all development side builds, this is now triggered by GitHub Actions that serve as a filter so only runs when infrastructure code is changed. See [deploy-infrastructure](../.github/workflows/deploy-infrastructure.yml)
 
 #### deployment
 Contains workflows, jobs, and commands for deploying the application on Cloud gov. Note: merges to develop now automatically trigger a develop deploy using [deploy-develop-on-merge](../.github/workflows/deploy-develop-on-merge.yml) and deploys to dev environments happen when a label is created on the PR using [deploy-on-label](../.github/workflows/deploy-on-label.yml)

diff --git a/.circleci/base_config.yml b/.circleci/base_config.yml
@@ -21,6 +21,18 @@ executors:
     resource_class: large
 
 parameters:
+  build_and_test_all:
+    type: boolean
+    default: false
+  build_and_test_backend:
+    type: boolean
+    default: false
+  build_and_test_frontend:
+    type: boolean
+    default: false
+  deploy_infrastructure:
+    type: boolean
+    default: false
   develop_branch_deploy:
     type: boolean
     default: false
@@ -36,3 +48,9 @@ parameters:
   target_env:
     type: string
     default: ''
+  triggered:
+    type: boolean
+    default: false
+  util_make_erd:
+    type: boolean
+    default: false
diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml
@@ -1,19 +1,67 @@
 # workflows:
-  build-and-test:
-    unless:
-      or:
-        - << pipeline.parameters.run_dev_deployment >>
-        - << pipeline.parameters.develop_branch_deploy >>
-        - << pipeline.parameters.run_owasp_scan >>
-        - << pipeline.parameters.run_nightly_owasp_scan >>
+  build-and-test-all:
+    when: << pipeline.parameters.build_and_test_all >>
     jobs:
       - secrets-check
+      - test-backend:
+          requires:
+            - secrets-check      
       - test-frontend:
           requires:
             - secrets-check
+      - test-e2e:
+          requires:
+            - secrets-check
+
+  ci-build-and-test-all:
+    jobs:
+      - secrets-check:
+          filters:
+            branches:
+              only:
+                - main
+                - master
+                - /^release.*/
       - test-backend:
+          filters:
+            branches:
+              only:
+                - main
+                - master
+                - /^release.*/
+          requires:
+            - secrets-check      
+      - test-frontend:
+          filters:
+            branches:
+              only:
+                - main
+                - master
+                - /^release.*/
           requires:
             - secrets-check
       - test-e2e:
+          filters:
+            branches:
+              only:
+                - main
+                - master
+                - /^release.*/
+          requires:
+            - secrets-check
+
+  build-and-test-backend:
+    when: << pipeline.parameters.build_and_test_backend >>
+    jobs:
+      - secrets-check
+      - test-backend:
+          requires:
+            - secrets-check
+
+  build-and-test-frontend:
+    when: << pipeline.parameters.build_and_test_frontend >>
+    jobs:
+      - secrets-check
+      - test-frontend:
           requires:
             - secrets-check
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -13,6 +13,18 @@ orbs:
 
 # parameters from github actions
 parameters:
+  build_and_test_all:
+    type: boolean
+    default: false
+  build_and_test_backend:
+    type: boolean
+    default: false
+  build_and_test_frontend:
+    type: boolean
+    default: false
+  deploy_infrastructure:
+    type: boolean
+    default: false
   develop_branch_deploy:
     type: boolean
     default: false
@@ -28,6 +40,12 @@ parameters:
   target_env:
     type: string
     default: ''
+  triggered:
+    type: boolean
+    default: false
+  util_make_erd:
+    type: boolean
+    default: false
 
 jobs:
   setup:
@@ -45,6 +63,23 @@ jobs:
 
 # our single workflow, that triggers the setup job defined above
 workflows:
-  setup:
+  github-triggered-setup:
+    when: << pipeline.parameters.triggered >>
+    jobs:
+      - setup:
+          filters:
+            branches:
+              ignore:
+                - main
+                - master
+                - /^release.*/
+  circle-ci-setup:
     jobs:
-      - setup
+      - setup:
+          filters:
+            branches:
+              only:
+                - main
+                - master
+                - /^release.*/      
+
diff --git a/.circleci/infrastructure/workflows.yml b/.circleci/infrastructure/workflows.yml
@@ -1,6 +1,10 @@
-#workflows:
+#workflows: 
   enable-versioning-for-s3-buckets:
-    unless: << pipeline.parameters.run_nightly_owasp_scan >>
+    when: 
+      or:
+        - << pipeline.parameters.deploy_infrastructure >>
+        - equal: [ 'main', << pipeline.git.branch >> ]
+        - equal: [ 'master', << pipeline.git.branch >> ]
     jobs:
       - enable-versioning:
           filters:

diff --git a/.github/workflows/build-all.yml b/.github/workflows/build-all.yml
@@ -0,0 +1,58 @@
+########################################################################### 
+# GitHub Action Workflow
+# On changes to scripts or changes to the pipeline code to any branch
+# besides develop, staging and master triggers the full build and test
+# pipeline.
+# 
+# NOTE: develop, staging(main) and master are skipped on the push because this
+#       would be redundant after running the full set of tests from the PR.
+#       See build-pr.yml for builds that run on code being merged into develop.
+#       See deploy-develop-on-merge.yml and make_erd for the workflow
+#       pipelines that run on merge to develop, staging, and master branches.
+#       HHS (main and master and release/**) branches build all only
+#       and are managed in CircleCI
+#
+# Step 0: Make changes on your branch to files in scripts/ .circleci or .github
+#         and push changes to your remote branch.
+#
+# Step 1: Makes a request to the V2 CircleCI API to initiate the project,
+#         which will filter based upon build_and_test_backend and 
+#         build_and_test_frontend to run the workflow/jobs listed here:
+#           build-and-test:[
+#             test-backend,
+#             test-frontend,
+#             test-e2e
+#           ]
+#
+#         Leverages the open source GitHub Action:
+#           https://github.com/promiseofcake/circleci-trigger-action
+###########################################################################
+name: Build and test All on push when scripts/commands change
+on:
+  push:
+    branches-ignore:
+      - develop
+      - main
+      - master
+      - 'release/**'
+    paths:
+      - 'scripts/**'
+      - '.circleci/**'
+      - '.github/**'
+jobs:
+  build_and_test_all:
+    runs-on: ubuntu-latest
+    name: Initiate deploy job in CircleCI
+    steps:
+      - uses: actions/checkout@v2
+      - name: Circle CI Deployment Trigger
+        id: curl-circle-ci
+        uses: promiseofcake/circleci-trigger-action@v1
+        with:
+          user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }}
+          project-slug: ${{ github.repository }}
+          branch: ${{ (github.event_name == 'pull_request') && github.head_ref || github.ref_name }}
+          payload: '{
+            "build_and_test_all": true,
+            "triggered": true
+          }'
diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml
@@ -0,0 +1,44 @@
+###########################################################################
+# GitHub Action Workflow
+# On push to any branch, triggers the back end build and test pipeline
+# if the tdrs-backend has changed.
+#
+# Step 0: make changes on your branch to non-documentation files in
+#         tdrs-backend and push changes to your remote branch
+#
+# Step 1: Makes a request to the V2 CircleCI API to initiate the project,
+#         which will filter based upon build_and_test_backend
+#         to run the workflow/jobs listed here:
+#           build-and-test:[
+#             test-backend,
+#             test-e2e
+#           ]
+#
+#         Leverages the open source GitHub Action:
+#           https://github.com/promiseofcake/circleci-trigger-action
+###########################################################################
+name: Build Only Backend When tdrs-backend/ Files Change
+on:
+  push:
+    paths: 'tdrs-backend/**'
+    branches-ignore:
+      - develop
+      - main
+      - master
+jobs:
+  build_and_test_backend:
+    runs-on: ubuntu-latest
+    name: Build and Test Backend
+    steps:
+    - uses: actions/checkout@v2
+    - name: Circle CI Deployment Trigger
+      id: curl-circle-ci
+      uses: promiseofcake/circleci-trigger-action@v1
+      with:
+        user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }}
+        project-slug: ${{ github.repository }}
+        branch: ${{ github.ref_name }}
+        payload: '{
+          "build_and_test_backend": true,
+          "triggered": true
+        }'
diff --git a/.github/workflows/build-frontend.yml b/.github/workflows/build-frontend.yml
@@ -0,0 +1,44 @@
+###########################################################################
+# GitHub Action Workflow
+# On push to any branch, triggers the front end build and test pipeline
+# if the tdrs-frontend has changed.
+#
+# Step 0: make changes on your branch to non-documentation files in
+#         tdrs-frontend and push changes to your remote branch
+#
+# Step 1: Makes a request to the V2 CircleCI API to initiate the project,
+#         which will filter based upon build_and_test_frontend 
+#         to run the workflow/jobs listed here:
+#           build-and-test:[
+#             test-frontend,
+#             test-e2e
+#           ]
+#
+#         Leverages the open source GitHub Action:
+#           https://github.com/promiseofcake/circleci-trigger-action
+###########################################################################
+name: Build Only Frontend When tdrs-frontend Files Change
+on:
+  push:
+    paths: 'tdrs-frontend/**'
+    branches-ignore:
+      - develop
+      - main
+      - master
+jobs:
+  build_and_test_frontend:
+    runs-on: ubuntu-latest
+    name: Build and Test Frontend
+    steps:
+    - uses: actions/checkout@v2
+    - name: Circle CI Deployment Trigger
+      id: curl-circle-ci
+      uses: promiseofcake/circleci-trigger-action@v1
+      with:
+        user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }}
+        project-slug: ${{ github.repository }}
+        branch: ${{ github.ref_name }}
+        payload: '{
+          "build_and_test_frontend": true,
+          "triggered": true
+        }'
diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml
@@ -0,0 +1,54 @@
+########################################################################### 
+# GitHub Action Workflow
+# On pull requests requesting review from individuals, besides staging, 
+# master, and release branches triggers the full build and test pipeline. 
+# 
+# NOTE: release branches, staging(main) and master are skipped because
+#       these branch builds are managed in CircleCI
+#
+# Step 0: make PR from your branch into develop, then select reviewers.
+#
+# Step 1: Makes a request to the V2 CircleCI API to initiate the project,
+#         which will filter based upon build_and_test_backend and 
+#         build_and_test_frontend to run the workflow/jobs listed here:
+#           build-and-test:[
+#             test-backend,
+#             test-frontend,
+#             test-e2e
+#           ]
+#
+#         Leverages the open source GitHub Action:
+#           https://github.com/promiseofcake/circleci-trigger-action
+###########################################################################
+name: Build and test All for PRs
+on:
+  pull_request:
+    branches-ignore: #handled in circleci
+      - main
+      - master
+      - 'release/**'
+    types: [review_requested, ready_for_review, synchronize]
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
+      - '**.txt'
+      - '.gitattributes'
+      - '.gitignore'
+      - 'LICENSE'
+jobs:
+  build_and_test_pr:
+    runs-on: ubuntu-latest
+    name: Initiate deploy job in CircleCI
+    steps:
+      - uses: actions/checkout@v2
+      - name: Circle CI Deployment Trigger
+        id: curl-circle-ci
+        uses: promiseofcake/circleci-trigger-action@v1
+        with:
+          user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }}
+          project-slug: ${{ github.repository }}
+          branch: ${{ (github.event_name == 'pull_request') && github.head_ref || github.ref_name }}
+          payload: '{
+            "build_and_test_all": true,
+            "triggered": true
+          }'