Add pull request build for Jan release

.github/readme.md

@@ -20,4 +20,5 @@ After the release is deployed on Prod
 ## Update frontend/package.json
 * version
 
-TBV
+
+

.github/workflows/branch-deploy-template.yaml

@@ -161,4 +161,30 @@ jobs:
               --set databaseServiceHostName=${{ inputs.database-service-host-name }} \
               --set rabbitmqVHost=${{ inputs.rabbitmq-vhost }} \
               -n ${{ secrets.namespace }} -f ./values${{ inputs.suffix }}.yaml tfrs-scan-handler${{ inputs.suffix }} .
-          fi
+          fi
+
+      - name: Deploy tfrs-scan-coordinator
+        shell: bash {0}
+        run: |
+          oc tag ${{ secrets.tools-namespace }}/tfrs-scan-hacoordinatorndler:build-${{ inputs.branch-name }} ${{ secrets.namespace }}/tfrs-scan-coordinator:${{ inputs.env-name }}-${{ inputs.branch-name }}
+          cd charts/tfrs-apps/charts/tfrs-scan-coordinator
+          helm status -n ${{ secrets.namespace }} tfrs-scan-coordinator${{ inputs.suffix }}
+          if [ $? -eq 0 ]; then
+            echo "tfrs-scan-coordinator${{ inputs.suffix }} release exists already"
+            helm upgrade \
+              --set scanCoordinatorImageTagName=${{ inputs.env-name }}-${{ inputs.branch-name }} \
+              --set suffix=${{ inputs.suffix }} \
+              --set namespace=${{ secrets.namespace }} \
+              --set envName=${{ inputs.env-Name }} \
+              --set rabbitmqVHost=${{ inputs.rabbitmq-vhost }} \
+              -n ${{ secrets.namespace }} -f ./values${{ inputs.suffix }}.yaml tfrs-scan-coordinator${{ inputs.suffix }} .
+          else
+            echo "tfrs-scan-coordinator${{ inputs.suffix }} release does not exist"
+            helm install \
+              --set scanCoordinatorImageTagName=${{ inputs.env-name }}-${{ inputs.branch-name }} \
+              --set suffix=${{ inputs.suffix }} \
+              --set namespace=${{ secrets.namespace }} \
+              --set envName=${{ inputs.env-Name }} \
+              --set rabbitmqVHost=${{ inputs.rabbitmq-vhost }} \
+              -n ${{ secrets.namespace }} -f ./values${{ inputs.suffix }}.yaml tfrs-scan-coordinator${{ inputs.suffix }} .
+          fi          

.github/workflows/branch-build-template.yaml → .github/workflows/build-template.yaml

@@ -1,10 +1,18 @@
 
+# This template supports both pr build and branch build
 name: Branch Build Template
 
 on:
   workflow_call:
     inputs:
-      branch-name: # sample value: main-release-jan-2024
+      # when build branch, the sample value is -main-release-jan-2024
+      # when build pull request, the sample value is -jan-2024
+      suffix:
+        required: true
+        type: string
+      # when build branch, the sample value is main-release-jan-2024
+      # when build pull request, the sample value is refs/pull/2024/head
+      checkout-ref:
         required: true
         type: string
     secrets:
@@ -31,7 +39,7 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v3.5.3
         with:
-          ref: ${{ inputs.branch-name }}
+          ref: ${{ inputs.checkout-ref }}
 
       - name: Log in to Openshift
         uses: redhat-actions/oc-login@v1.2
@@ -45,13 +53,13 @@ jobs:
         run: |
           cd openshift-v4/templates/backend
           oc process -f ./backend-bc.yaml NAME=tfrs \
-            SUFFIX=-build-${{ inputs.branch-name}} \
-            VERSION=build-${{ inputs.branch-name }} \
+            SUFFIX=-build${{ inputs.suffix}} \
+            VERSION=build${{ inputs.suffix }} \
             GIT_URL=${{ env.GIT_URL }} \
-            GIT_REF=${{ inputs.branch-name }} \
+            GIT_REF=${{ inputs.checkout-ref }} \
             | oc apply --wait=true -f - -n ${{ secrets.tools-namespace }}
-          oc cancel-build bc/tfrs-backend-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} || true
-          oc start-build --wait=true tfrs-backend-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }}
+          oc cancel-build bc/tfrs-backend-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} || true
+          oc start-build --wait=true tfrs-backend-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }}
 
   build-frontend:
 
@@ -64,7 +72,7 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v3.5.3
         with:
-          ref: ${{ inputs.branch-name }}
+          ref: ${{ inputs.checkout-ref }}
 
       - name: Log in to Openshift
         uses: redhat-actions/oc-login@v1.2
@@ -78,17 +86,18 @@ jobs:
         run: |
           cd openshift-v4/templates/frontend
           oc process -f ./frontend-bc-docker.yaml NAME=tfrs \
-            SUFFIX=-build-${{ inputs.branch-name}} \
-            VERSION=build-${{ inputs.branch-name }} \
+            SUFFIX=-build${{ inputs.suffix}} \
+            VERSION=build${{ inputs.suffix }} \
             GIT_URL=${{ env.GIT_URL }} \
-            GIT_REF=${{ inputs.branch-name }} \
+            GIT_REF=${{ inputs.checkout-ref }} \
             | oc apply --wait=true -f - -n ${{ secrets.tools-namespace }}
-          oc cancel-build bc/tfrs-frontend-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} || true
-          oc start-build --wait=true tfrs-frontend-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }}
+          oc cancel-build bc/tfrs-frontend-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} || true
+          oc start-build --wait=true tfrs-frontend-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }}
 
   build-celery:
 
     name: Build TFRS Celery on Openshift
+    needs: [build-frontend, build-backend]
     runs-on: ubuntu-latest
     timeout-minutes: 60
 
@@ -97,7 +106,7 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v3.5.3
         with:
-          ref: ${{ inputs.branch-name }}
+          ref: ${{ inputs.checkout-ref }}
 
       - name: Log in to Openshift
         uses: redhat-actions/oc-login@v1.2
@@ -110,18 +119,21 @@ jobs:
       - name: Build TFRS Celery
         run: |
           cd openshift-v4/templates/celery
-          oc process -f ./celery-bc.yaml NAME=tfrs \
-            SUFFIX=-build-${{ inputs.branch-name}} \
-            VERSION=build-${{ inputs.branch-name }} \
+          pwd
+          ls -l
+          oc process -f ./celery-bc-docker.yaml NAME=tfrs \
+            SUFFIX=-build${{ inputs.suffix}} \
+            VERSION=build${{ inputs.suffix }} \
             GIT_URL=${{ env.GIT_URL }} \
-            RELEASE_BRANCH=${{ inputs.branch-name}} \
+            GIT_REF=${{ inputs.checkout-ref }} \
             | oc apply --wait=true -f - -n ${{ secrets.tools-namespace }}
-          oc cancel-build bc/tfrs-celery-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} || true
-          oc start-build --wait=true tfrs-celery-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }}
+          oc cancel-build bc/tfrs-celery-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} || true
+          oc start-build --wait=true tfrs-celery-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }}
 
   build-scan-coordinator:
 
     name: Build TFRS Scan Coordinator on Openshift
+    needs: [build-frontend, build-backend]
     runs-on: ubuntu-latest
     timeout-minutes: 60
 
@@ -130,7 +142,7 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v3.5.3
         with:
-          ref: ${{ inputs.branch-name }}
+          ref: ${{ inputs.checkout-ref }}
 
       - name: Log in to Openshift
         uses: redhat-actions/oc-login@v1.2
@@ -144,17 +156,18 @@ jobs:
         run: |
           cd openshift-v4/templates/scan-coordinator
           oc process -f ./scan-coordinator-bc.yaml NAME=tfrs \
-            SUFFIX=-build-${{ inputs.branch-name}} \
-            VERSION=build-${{ inputs.branch-name }} \
+            SUFFIX=-build${{ inputs.suffix}} \
+            VERSION=build${{ inputs.suffix }} \
             GIT_URL=${{ env.GIT_URL }} \
-            GIT_REF=${{ inputs.branch-name }} \
+            GIT_REF=${{ inputs.checkout-ref }} \
             | oc apply --wait=true -f - -n ${{ secrets.tools-namespace }}
-          oc cancel-build bc/tfrs-scan-coordinator-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} || true
-          oc start-build --wait=true tfrs-scan-coordinator-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }}          
+          oc cancel-build bc/tfrs-scan-coordinator-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} || true
+          oc start-build --wait=true tfrs-scan-coordinator-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }}          
 
   build-scan-handler:
 
     name: Build TFRS Scan Handler on Openshift
+    needs: [build-scan-coordinator, build-celery]
     runs-on: ubuntu-latest
     timeout-minutes: 60
 
@@ -163,7 +176,7 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v3.5.3
         with:
-          ref: ${{ inputs.branch-name }}
+          ref: ${{ inputs.checkout-ref }}
 
       - name: Log in to Openshift
         uses: redhat-actions/oc-login@v1.2
@@ -176,10 +189,11 @@ jobs:
       - name: Build TFRS Scan-Handler
         run: |
           cd openshift-v4/templates/scan-handler
-          oc process -f ./scan-handler-bc.yaml NAME=tfrs \
-            SUFFIX=-build-${{ inputs.branch-name}} \
-            VERSION=build-${{ inputs.branch-name }} \
-            RELEASE_BRANCH=${{ inputs.branch-name }} \
+          oc process -f ./scan-handler-bc-docker.yaml NAME=tfrs \
+            SUFFIX=-build${{ inputs.suffix}} \
+            VERSION=build${{ inputs.suffix }} \
+            GIT_URL=${{ env.GIT_URL }} \
+            GIT_REF=${{ inputs.checkout-ref }} \
             | oc apply --wait=true -f - -n ${{ secrets.tools-namespace }}
-          oc cancel-build bc/tfrs-scan-handler-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} || true
-          oc start-build --wait=true tfrs-scan-handler-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }}                    
+          oc cancel-build bc/tfrs-scan-handler-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} || true
+          oc start-build --wait=true tfrs-scan-handler-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }}   

.github/workflows/dev-jan-release.yaml

@@ -65,12 +65,15 @@ jobs:
           VALIDATE_PYTHON_PYLINT: true
           LOG_LEVEL: WARN
 
+  # when build branch, the suffix sample is -main-release-jan-2024
+  #       the  checkout-ref sample is main-release-jan-2024
   build:
     name: Build
     needs: [unit-test, lint]
-    uses: ./.github/workflows/branch-build-template.yaml
+    uses: ./.github/workflows/build-template.yaml
     with:
-      branch-name: ${{ github.ref_name }}
+      suffix: -${{ github.ref_name }}
+      checkout-ref: ${{ github.ref_name }}
     secrets:
       tools-namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-tools
       openshift-server: ${{ secrets.OPENSHIFT_SERVER }}

.github/workflows/pr-dev-cicd.yaml

@@ -0,0 +1,54 @@
+# Please refer to ./readme.md for how to build single pull request
+
+# Update this workflow name per pull request
+name: TFRS Dev Jan PR CICD
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [opened, edited, synchronize, reopened]
+    branches:
+      - 'main-release-jan-2024'
+
+jobs:
+
+  setup-database:
+    if: endsWith( github.event.pull_request.title, 'build-on-dev' )
+    uses: ./.github/workflows/pr-dev-database-template.yaml
+    with:
+      pr-number: ${{ github.event.pull_request.number }}
+      dev-suffix: -jan-${{ github.event.pull_request.number }}
+    secrets:
+      dev-namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-dev
+      tfrs-dev-username: ${{ secrets.TFRS_DEV_USERNAME }}
+      tfrs-dev-password: ${{ secrets.TFRS_DEV_PASSWORD }}
+      openshift-server: ${{ secrets.OPENSHIFT_SERVER }}
+      openshift-token: ${{ secrets.OPENSHIFT_TOKEN }}
+
+  # when build pull reuqest, the suffix sample is -jan-1234
+  #       the  checkout-ref is in the format of refs/pull/1234/head
+  build:
+    if: endsWith( github.event.pull_request.title, 'build-on-dev' )
+    name: Build Pull Request
+    uses: ./.github/workflows/build-template.yaml
+    with:
+      suffix: -jan-${{ github.event.pull_request.number }}
+      checkout-ref: refs/pull/${{ github.event.pull_request.number }}/head
+    secrets:
+      tools-namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-tools
+      openshift-server: ${{ secrets.OPENSHIFT_SERVER }}
+      openshift-token: ${{ secrets.OPENSHIFT_TOKEN }}      
+
+  deploy:
+    if: endsWith( github.event.pull_request.title, 'build-on-dev' )
+    needs: [setup-database, build]
+    uses: ./.github/workflows/pr-dev-deploy-template.yaml
+    with:
+      suffix: -jan-${{ github.event.pull_request.number }}
+      checkout-ref: refs/pull/${{ github.event.pull_request.number }}/head
+      database-service-host-name: tfrs-spilo-jan-${{ github.event.pull_request.number }}
+    secrets:
+      tools-namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-tools
+      namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-dev
+      openshift-server: ${{ secrets.OPENSHIFT_SERVER }}
+      openshift-token: ${{ secrets.OPENSHIFT_TOKEN }}
+

.github/workflows/pr-dev-database-template.yaml

@@ -0,0 +1,69 @@
+name: PR Dev Database Template
+
+on:
+  workflow_call:
+    inputs:
+      # pull request number
+      pr-number:
+        required: true
+        type: string
+      # the suffix will be appended to tfrs-spilo, same values: -1234, -jan-1242
+      dev-suffix:
+        required: true
+        type: string        
+    secrets:
+      dev-namespace:
+        required: true
+      tfrs-dev-username:
+        required: true
+      tfrs-dev-password:
+        required: true       
+      openshift-server:
+         required: true 
+      openshift-token:   
+         required: true       
+
+jobs:
+
+  database:
+
+    name: Start Database
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    steps:
+
+      - name: Check out repository
+        uses: actions/checkout@v3
+        with:
+          ref: refs/pull/${{ inputs.pr-number }}/head
+
+      - name: Log in to Openshift
+        uses: redhat-actions/oc-login@v1.2
+        with:
+          openshift_server_url: ${{ secrets.openshift-server }}
+          openshift_token: ${{ secrets.openshift-token }}
+          insecure_skip_tls_verify: true
+          namespace: ${{ secrets.dev-namespace }}    
+
+      - name: Setup Database
+        shell: bash {0}
+        run: |
+          cd charts/tfrs-spilo
+          helm dependency build
+          helm status -n ${{ secrets.dev-namespace }} tfrs-spilo${{ inputs.dev-suffix }}
+          if [ $? -eq 0 ]; then
+            echo "tfrs-spilo${{ inputs.dev-suffix }} exists already"
+          else
+            echo "Installing tfrs-spilo${{ inputs.dev-suffix }}"
+            helm install -n ${{ secrets.dev-namespace }} -f ./values-dev.yaml --wait tfrs-spilo${{ inputs.dev-suffix }} .
+            oc -n ${{ secrets.dev-namespace }} wait --for=condition=Ready pod/tfrs-spilo${{ inputs.dev-suffix }}-0
+            oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "create user \"${{ secrets.tfrs-dev-username }}\" WITH PASSWORD '${{ secrets.tfrs-dev-password }}'" || true
+            oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "create database tfrs owner \"${{ secrets.tfrs-dev-username }}\" ENCODING 'UTF8' LC_COLLATE = 'en_US.UTF-8' LC_CTYPE = 'en_US.UTF-8'" || true
+            oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "ALTER SYSTEM SET log_filename='postgresql-%H.log'" || true
+            oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "ALTER SYSTEM SET log_connections='off'" || true
+            oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "ALTER SYSTEM SET log_disconnections='off'" || true
+            oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "ALTER SYSTEM SET log_checkpoints='off'" || true
+            oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "select pg_reload_conf()" || true
+          fi
+