From ab0966005a875f1ccb2c007d6d905d570998a1a7 Mon Sep 17 00:00:00 2001
From: George Hudson <georgehudson78@gmail.com>
Date: Tue, 27 Jun 2023 11:40:22 -0600
Subject: [PATCH] pipeline filtering (#2538)

* pipeline changes that filter based on paths and branches. circle ci tracks specified branches in order to keep current functionality on HHS side.

* updated syntax to be in line with build-all.yml

* removed comma

* WIP build flow docs

* added Architecture Decision Record for the change to pipeline workflows

* corrected file type of doc to .md

---------

Co-authored-by: George Hudson <ghudson@teamraft.com>
Co-authored-by: Andrew <84722778+andrew-jameson@users.noreply.github.com>
---
 .circleci/README.md                           |  4 +-
 .circleci/base_config.yml                     | 18 ++++++
 .circleci/build-and-test/workflows.yml        | 62 ++++++++++++++++---
 .circleci/config.yml                          | 39 +++++++++++-
 .circleci/infrastructure/workflows.yml        |  8 ++-
 .github/workflows/build-all.yml               | 58 +++++++++++++++++
 .github/workflows/build-backend.yml           | 44 +++++++++++++
 .github/workflows/build-frontend.yml          | 44 +++++++++++++
 .github/workflows/build-pr.yml                | 54 ++++++++++++++++
 .github/workflows/deploy-develop-on-merge.yml |  2 +-
 .github/workflows/deploy-infrastructure.yml   | 46 ++++++++++++++
 .../020-pipeline-build-flow.md                | 44 +++++++++++++
 .../Technical-Documentation/github-actions.md |  5 ++
 13 files changed, 414 insertions(+), 14 deletions(-)
 create mode 100644 .github/workflows/build-all.yml
 create mode 100644 .github/workflows/build-backend.yml
 create mode 100644 .github/workflows/build-frontend.yml
 create mode 100644 .github/workflows/build-pr.yml
 create mode 100644 .github/workflows/deploy-infrastructure.yml
 create mode 100644 docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md
 create mode 100644 docs/Technical-Documentation/github-actions.md

diff --git a/.circleci/README.md b/.circleci/README.md
index 39e484871c..1b0c4c0311 100644
--- a/.circleci/README.md
+++ b/.circleci/README.md
@@ -14,10 +14,10 @@ This script will generate a complete config for building, testing, and deploying
 ### Directory structure
 
 #### build-and-test
-Contains workflows, jobs, and commands for building and testing the application.
+Contains workflows, jobs, and commands for building and testing the application. For all development side builds, these are now triggered by GitHub Actions that serve as a filter so only the code that's changed is tested. See [build-all](../.github/workflows/build-all.yml), [build-backend](../.github/workflows/build-backend.yml), and [build-frontend](../.github/workflows/build-frontend.yml)
 
 #### infrastructure
-Contains workflows, jobs, and commands for setting up the infrastructure on Cloud gov.
+Contains workflows, jobs, and commands for setting up the infrastructure on Cloud gov. For all development side builds, this is now triggered by GitHub Actions that serve as a filter so only runs when infrastructure code is changed. See [deploy-infrastructure](../.github/workflows/deploy-infrastructure.yml)
 
 #### deployment
 Contains workflows, jobs, and commands for deploying the application on Cloud gov. Note: merges to develop now automatically trigger a develop deploy using [deploy-develop-on-merge](../.github/workflows/deploy-develop-on-merge.yml) and deploys to dev environments happen when a label is created on the PR using [deploy-on-label](../.github/workflows/deploy-on-label.yml)
diff --git a/.circleci/base_config.yml b/.circleci/base_config.yml
index eb4b9af15c..d722abecf1 100644
--- a/.circleci/base_config.yml
+++ b/.circleci/base_config.yml
@@ -21,6 +21,18 @@ executors:
     resource_class: large
 
 parameters:
+  build_and_test_all:
+    type: boolean
+    default: false
+  build_and_test_backend:
+    type: boolean
+    default: false
+  build_and_test_frontend:
+    type: boolean
+    default: false
+  deploy_infrastructure:
+    type: boolean
+    default: false
   develop_branch_deploy:
     type: boolean
     default: false
@@ -36,3 +48,9 @@ parameters:
   target_env:
     type: string
     default: ''
+  triggered:
+    type: boolean
+    default: false
+  util_make_erd:
+    type: boolean
+    default: false
diff --git a/.circleci/build-and-test/workflows.yml b/.circleci/build-and-test/workflows.yml
index 9122ab1666..7c0e559b0f 100644
--- a/.circleci/build-and-test/workflows.yml
+++ b/.circleci/build-and-test/workflows.yml
@@ -1,19 +1,67 @@
 # workflows:
-  build-and-test:
-    unless:
-      or:
-        - << pipeline.parameters.run_dev_deployment >>
-        - << pipeline.parameters.develop_branch_deploy >>
-        - << pipeline.parameters.run_owasp_scan >>
-        - << pipeline.parameters.run_nightly_owasp_scan >>
+  build-and-test-all:
+    when: << pipeline.parameters.build_and_test_all >>
     jobs:
       - secrets-check
+      - test-backend:
+          requires:
+            - secrets-check      
       - test-frontend:
           requires:
             - secrets-check
+      - test-e2e:
+          requires:
+            - secrets-check
+
+  ci-build-and-test-all:
+    jobs:
+      - secrets-check:
+          filters:
+            branches:
+              only:
+                - main
+                - master
+                - /^release.*/
       - test-backend:
+          filters:
+            branches:
+              only:
+                - main
+                - master
+                - /^release.*/
+          requires:
+            - secrets-check      
+      - test-frontend:
+          filters:
+            branches:
+              only:
+                - main
+                - master
+                - /^release.*/
           requires:
             - secrets-check
       - test-e2e:
+          filters:
+            branches:
+              only:
+                - main
+                - master
+                - /^release.*/
+          requires:
+            - secrets-check
+
+  build-and-test-backend:
+    when: << pipeline.parameters.build_and_test_backend >>
+    jobs:
+      - secrets-check
+      - test-backend:
+          requires:
+            - secrets-check
+
+  build-and-test-frontend:
+    when: << pipeline.parameters.build_and_test_frontend >>
+    jobs:
+      - secrets-check
+      - test-frontend:
           requires:
             - secrets-check
diff --git a/.circleci/config.yml b/.circleci/config.yml
index ea0ddd1c49..5d0be7af65 100755
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -13,6 +13,18 @@ orbs:
 
 # parameters from github actions
 parameters:
+  build_and_test_all:
+    type: boolean
+    default: false
+  build_and_test_backend:
+    type: boolean
+    default: false
+  build_and_test_frontend:
+    type: boolean
+    default: false
+  deploy_infrastructure:
+    type: boolean
+    default: false
   develop_branch_deploy:
     type: boolean
     default: false
@@ -28,6 +40,12 @@ parameters:
   target_env:
     type: string
     default: ''
+  triggered:
+    type: boolean
+    default: false
+  util_make_erd:
+    type: boolean
+    default: false
 
 jobs:
   setup:
@@ -45,6 +63,23 @@ jobs:
 
 # our single workflow, that triggers the setup job defined above
 workflows:
-  setup:
+  github-triggered-setup:
+    when: << pipeline.parameters.triggered >>
+    jobs:
+      - setup:
+          filters:
+            branches:
+              ignore:
+                - main
+                - master
+                - /^release.*/
+  circle-ci-setup:
     jobs:
-      - setup
+      - setup:
+          filters:
+            branches:
+              only:
+                - main
+                - master
+                - /^release.*/      
+      
diff --git a/.circleci/infrastructure/workflows.yml b/.circleci/infrastructure/workflows.yml
index 46f29aec7a..9cb85e8ed8 100644
--- a/.circleci/infrastructure/workflows.yml
+++ b/.circleci/infrastructure/workflows.yml
@@ -1,6 +1,10 @@
-#workflows:
+#workflows: 
   enable-versioning-for-s3-buckets:
-    unless: << pipeline.parameters.run_nightly_owasp_scan >>
+    when: 
+      or:
+        - << pipeline.parameters.deploy_infrastructure >>
+        - equal: [ 'main', << pipeline.git.branch >> ]
+        - equal: [ 'master', << pipeline.git.branch >> ]
     jobs:
       - enable-versioning:
           filters:
diff --git a/.github/workflows/build-all.yml b/.github/workflows/build-all.yml
new file mode 100644
index 0000000000..c03d344abb
--- /dev/null
+++ b/.github/workflows/build-all.yml
@@ -0,0 +1,58 @@
+########################################################################### 
+# GitHub Action Workflow
+# On changes to scripts or changes to the pipeline code to any branch
+# besides develop, staging and master triggers the full build and test
+# pipeline.
+# 
+# NOTE: develop, staging(main) and master are skipped on the push because this
+#       would be redundant after running the full set of tests from the PR.
+#       See build-pr.yml for builds that run on code being merged into develop.
+#       See deploy-develop-on-merge.yml and make_erd for the workflow
+#       pipelines that run on merge to develop, staging, and master branches.
+#       HHS (main and master and release/**) branches build all only
+#       and are managed in CircleCI
+#
+# Step 0: Make changes on your branch to files in scripts/ .circleci or .github
+#         and push changes to your remote branch.
+#
+# Step 1: Makes a request to the V2 CircleCI API to initiate the project,
+#         which will filter based upon build_and_test_backend and 
+#         build_and_test_frontend to run the workflow/jobs listed here:
+#           build-and-test:[
+#             test-backend,
+#             test-frontend,
+#             test-e2e
+#           ]
+#
+#         Leverages the open source GitHub Action:
+#           https://github.com/promiseofcake/circleci-trigger-action
+###########################################################################
+name: Build and test All on push when scripts/commands change
+on:
+  push:
+    branches-ignore:
+      - develop
+      - main
+      - master
+      - 'release/**'
+    paths:
+      - 'scripts/**'
+      - '.circleci/**'
+      - '.github/**'
+jobs:
+  build_and_test_all:
+    runs-on: ubuntu-latest
+    name: Initiate deploy job in CircleCI
+    steps:
+      - uses: actions/checkout@v2
+      - name: Circle CI Deployment Trigger
+        id: curl-circle-ci
+        uses: promiseofcake/circleci-trigger-action@v1
+        with:
+          user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }}
+          project-slug: ${{ github.repository }}
+          branch: ${{ (github.event_name == 'pull_request') && github.head_ref || github.ref_name }}
+          payload: '{
+            "build_and_test_all": true,
+            "triggered": true
+          }'
diff --git a/.github/workflows/build-backend.yml b/.github/workflows/build-backend.yml
new file mode 100644
index 0000000000..26ef5c03eb
--- /dev/null
+++ b/.github/workflows/build-backend.yml
@@ -0,0 +1,44 @@
+###########################################################################
+# GitHub Action Workflow
+# On push to any branch, triggers the back end build and test pipeline
+# if the tdrs-backend has changed.
+#
+# Step 0: make changes on your branch to non-documentation files in
+#         tdrs-backend and push changes to your remote branch
+#
+# Step 1: Makes a request to the V2 CircleCI API to initiate the project,
+#         which will filter based upon build_and_test_backend
+#         to run the workflow/jobs listed here:
+#           build-and-test:[
+#             test-backend,
+#             test-e2e
+#           ]
+#
+#         Leverages the open source GitHub Action:
+#           https://github.com/promiseofcake/circleci-trigger-action
+###########################################################################
+name: Build Only Backend When tdrs-backend/ Files Change
+on:
+  push:
+    paths: 'tdrs-backend/**'
+    branches-ignore:
+      - develop
+      - main
+      - master
+jobs:
+  build_and_test_backend:
+    runs-on: ubuntu-latest
+    name: Build and Test Backend
+    steps:
+    - uses: actions/checkout@v2
+    - name: Circle CI Deployment Trigger
+      id: curl-circle-ci
+      uses: promiseofcake/circleci-trigger-action@v1
+      with:
+        user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }}
+        project-slug: ${{ github.repository }}
+        branch: ${{ github.ref_name }}
+        payload: '{
+          "build_and_test_backend": true,
+          "triggered": true
+        }'
diff --git a/.github/workflows/build-frontend.yml b/.github/workflows/build-frontend.yml
new file mode 100644
index 0000000000..b9b60a9141
--- /dev/null
+++ b/.github/workflows/build-frontend.yml
@@ -0,0 +1,44 @@
+###########################################################################
+# GitHub Action Workflow
+# On push to any branch, triggers the front end build and test pipeline
+# if the tdrs-frontend has changed.
+#
+# Step 0: make changes on your branch to non-documentation files in
+#         tdrs-frontend and push changes to your remote branch
+#
+# Step 1: Makes a request to the V2 CircleCI API to initiate the project,
+#         which will filter based upon build_and_test_frontend 
+#         to run the workflow/jobs listed here:
+#           build-and-test:[
+#             test-frontend,
+#             test-e2e
+#           ]
+#
+#         Leverages the open source GitHub Action:
+#           https://github.com/promiseofcake/circleci-trigger-action
+###########################################################################
+name: Build Only Frontend When tdrs-frontend Files Change
+on:
+  push:
+    paths: 'tdrs-frontend/**'
+    branches-ignore:
+      - develop
+      - main
+      - master
+jobs:
+  build_and_test_frontend:
+    runs-on: ubuntu-latest
+    name: Build and Test Frontend
+    steps:
+    - uses: actions/checkout@v2
+    - name: Circle CI Deployment Trigger
+      id: curl-circle-ci
+      uses: promiseofcake/circleci-trigger-action@v1
+      with:
+        user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }}
+        project-slug: ${{ github.repository }}
+        branch: ${{ github.ref_name }}
+        payload: '{
+          "build_and_test_frontend": true,
+          "triggered": true
+        }'
diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml
new file mode 100644
index 0000000000..fa54a2a097
--- /dev/null
+++ b/.github/workflows/build-pr.yml
@@ -0,0 +1,54 @@
+########################################################################### 
+# GitHub Action Workflow
+# On pull requests requesting review from individuals, besides staging, 
+# master, and release branches triggers the full build and test pipeline. 
+# 
+# NOTE: release branches, staging(main) and master are skipped because
+#       these branch builds are managed in CircleCI
+#
+# Step 0: make PR from your branch into develop, then select reviewers.
+#
+# Step 1: Makes a request to the V2 CircleCI API to initiate the project,
+#         which will filter based upon build_and_test_backend and 
+#         build_and_test_frontend to run the workflow/jobs listed here:
+#           build-and-test:[
+#             test-backend,
+#             test-frontend,
+#             test-e2e
+#           ]
+#
+#         Leverages the open source GitHub Action:
+#           https://github.com/promiseofcake/circleci-trigger-action
+###########################################################################
+name: Build and test All for PRs
+on:
+  pull_request:
+    branches-ignore: #handled in circleci
+      - main
+      - master
+      - 'release/**'
+    types: [review_requested, ready_for_review, synchronize]
+    paths-ignore:
+      - 'docs/**'
+      - '**.md'
+      - '**.txt'
+      - '.gitattributes'
+      - '.gitignore'
+      - 'LICENSE'
+jobs:
+  build_and_test_pr:
+    runs-on: ubuntu-latest
+    name: Initiate deploy job in CircleCI
+    steps:
+      - uses: actions/checkout@v2
+      - name: Circle CI Deployment Trigger
+        id: curl-circle-ci
+        uses: promiseofcake/circleci-trigger-action@v1
+        with:
+          user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }}
+          project-slug: ${{ github.repository }}
+          branch: ${{ (github.event_name == 'pull_request') && github.head_ref || github.ref_name }}
+          payload: '{
+            "build_and_test_all": true,
+            "triggered": true
+          }'
\ No newline at end of file
diff --git a/.github/workflows/deploy-develop-on-merge.yml b/.github/workflows/deploy-develop-on-merge.yml
index 80ad9b3088..a9df826455 100644
--- a/.github/workflows/deploy-develop-on-merge.yml
+++ b/.github/workflows/deploy-develop-on-merge.yml
@@ -24,7 +24,7 @@ on:
   push:
     branches:
     - develop
-    paths_ignore:
+    paths-ignore:
     - 'docs/**'
     - '**.md'
     - '**.txt'
diff --git a/.github/workflows/deploy-infrastructure.yml b/.github/workflows/deploy-infrastructure.yml
new file mode 100644
index 0000000000..e5eeeb611a
--- /dev/null
+++ b/.github/workflows/deploy-infrastructure.yml
@@ -0,0 +1,46 @@
+###########################################################################
+# GitHub Action Workflow
+# On push changing terraform files or infrastructure pipelines, triggers the 
+# terraform deploy pipeline for the appropriate cf space.
+#
+# Step 0: make changes to non-documentation files in terraform/ or
+#         .circleci/infrastructure/ and push/merge changes.
+#
+# Step 1: Makes a request to the V2 CircleCI API to initiate the project,
+#         which will filter based upon terraform: true flag 
+#         to run the workflow/jobs listed here:
+#           build-and-test:[
+#             enable-versioning-for-s3-buckets
+#           ]
+#
+#         Leverages the open source GitHub Action:
+#           https://github.com/promiseofcake/circleci-trigger-action
+###########################################################################
+name: Run Infrastructure Pipeline When Terraform or Infrastructure Files Change
+on:
+  push:
+    branches-ignore: #handled in CircleCI
+      - main
+      - master
+      - 'release/**'
+    paths:
+      - 'terraform/**'
+      - '.circleci/infrastructure/**'
+jobs:
+  run_infrastructure_deployment:
+    runs-on: ubuntu-latest
+    name: Deploy Infrastructure
+    steps:
+      - uses: actions/checkout@v2
+      - name: Circle CI Deployment Trigger
+        id: curl-circle-ci
+        uses: promiseofcake/circleci-trigger-action@v1
+        with:
+          user-token: ${{ secrets.CIRCLE_CI_V2_TOKEN }}
+          project-slug: ${{ github.repository }}
+          branch: ${{ github.ref_name }}
+          payload: '{
+            "deploy_infrastructure": true,
+            "triggered": true
+          }'
+          
\ No newline at end of file
diff --git a/docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md b/docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md
new file mode 100644
index 0000000000..0e5f9b17ce
--- /dev/null
+++ b/docs/Technical-Documentation/Architecture-Decision-Record/020-pipeline-build-flow.md
@@ -0,0 +1,44 @@
+# 20. Pipeline Build Flow
+
+Date: 2023-06-07 (_Updated 2023-06-13_)
+
+## Status
+
+Pending
+
+## Context
+
+We use [CircleCI](https://app.circleci.com/pipelines/github/raft-tech/TANF-app)] as our primary pipeline build tool.  On the HHS side, the standard pipeline, build all and deploy ran by CircleCI is sufficiently granular to meet all our build needs. However, on the dev Raft-TANF side, where work can often be held up by waiting for build and test pipelines, it's useful to reduce build times by filtering which workflows run based upon which code is changing. In order to do this, GitHub Actions is leveraged to kick off different CircleCI build workflows since Actions has more granular control over what paths are changed. 
+
+These pipelines are closely tied to the [Git workflow](./009-git-workflow.md) and should run build and test as expected based on what code changed, and still deploy to the correct CF spaces based upon the Decision tree in the [Deployment Flow](./008-deployment-flow.md)   
+
+## Build Logic
+
+For all release branches, the main, and the master branch, CircleCI should run the full build, infrastructure deploy, and app deployment.
+
+for feature/development branches, only the build and test pertainint to the code changed should be built.
+Front end tests should be run if /tdrs-frontent changes
+Back end tests should be run if /tdrs-backend changes
+the entire build and test all should run if anything pertaining to the pipeline changes
+infrastructure deploy should run if /terraform or infrastructure deploy pipeline changes
+
+Once a pull request is flagged as ready for review and/or has reviewers assigned, a full build and test all should be run (and tests must pass before merge to develop)
+
+Develop merges trigger a deploy to the develop CF space, and then a special integration end-2-end test that tests against the real development environment instead of a simulated environment on the CircleCI build servers. 
+
+## Consequences
+
+**Pros**
+* reduce time of build and tests by only running the appropriate workflows.
+* only run infrastructure deploys when changes are made to infrastructure code.
+* only run end-2-end tests when code is ready to be reviewed.
+* only run full integration end-2-end testing when the develop environment assets are updated.
+
+**Risks**
+* Increased pipeline logic complexity
+
+## Notes
+
+- For the nuanced build triggers, CircleCI documentation recommends using Actions to trigger CircleCI builds and send different flags to tell which workflows should run. See this [blog](https://circleci.com/blog/trigger-circleci-pipeline-github-action/) for details, though we use [promiseofcake/circleci-trigger-action@v](https://github.com/promiseofcake/circleci-trigger-action) plugin vs circleci/trigger_circleci_pipeline@v1.0
+
+- This could be an argument for future complete pipeline switchover to GitHub Actions in order to reduce pipeline complexity, while maintaining the desired build granularity.
\ No newline at end of file
diff --git a/docs/Technical-Documentation/github-actions.md b/docs/Technical-Documentation/github-actions.md
new file mode 100644
index 0000000000..05839498a4
--- /dev/null
+++ b/docs/Technical-Documentation/github-actions.md
@@ -0,0 +1,5 @@
+# How We Use GitHub Actions
+For now, the only use case we have for GitHub Actions is to help up trigger CircleCI builds the way we want to. This is actually the preferred method CircleCI advises for branch, path, pull-request, and labelled filtering and job triggering. See this [blog](https://circleci.com/blog/trigger-circleci-pipeline-github-action/) for details, though we use [promiseofcake/circleci-trigger-action@v](https://github.com/promiseofcake/circleci-trigger-action) plugin vs circleci/trigger_circleci_pipeline@v1.0
+
+## Path Filtering
+We use Actions to filter which workflows are getting run by CircleCI by sending different flags to CircleCI through the promiseofcake CircleCI API trigger. See the individual files in [.github](../../.github/) for detailed instructions for how to use each.