Add slack alerts for scheduled job submissions

Earth-Information-System · Sep 27, 2024 · ef57491 · ef57491
1 parent d5df8d4
commit ef57491
Show file tree

Hide file tree

Showing 5 changed files with 179 additions and 135 deletions.
diff --git a/.github/workflows/schedule-borealna-nrt-v3.yaml b/.github/workflows/schedule-borealna-nrt-v3.yaml
@@ -3,7 +3,7 @@ name: borealna nrt v3
 on:
   workflow_dispatch:
   schedule:
-    - cron:  '25 15,23 * * *'
+    - cron: "25 15,23 * * *"
 
 jobs:
   run_dps_job_v3:
@@ -13,35 +13,44 @@ jobs:
     runs-on: ubuntu-latest
     environment: production
     steps:
-    - name: checkout repository
-      uses: actions/checkout@v2
-
-    - name: pip install fireatlas
-      run: |
-        pip install .
-
-    - name: get docker GID and set DOCKER_GID environment variable
-      run: |
-        echo "DOCKER_GID=$(getent group docker | cut -d: -f3)" >> $GITHUB_ENV
-
-    - name: get current user UID and set UID environment variable
-      run: |
-        echo "UID=$(id -u)" >> $GITHUB_ENV
-
-    - name: print DOCKER_GID and UID
-      run: |
-        echo "Docker GID: $DOCKER_GID"
-        echo "User UID: $UID"
-
-    - name: kick off the DPS job
-      uses: Earth-Information-System/fireatlas/.github/actions/run-dps-job-v3@conus-dps
-      with:
-        algo_name: eis-feds-dask-coordinator-v3
-        github_ref: 1.2.3
-        username: gcorradini
-        queue: maap-dps-eis-worker-128gb
-        maap_image_env: ubuntu
-        maap_pgt_secret: ${{ secrets.MAAP_PGT }}
-        json_params: '{"regnm": "BorealNA", "bbox": "[-169, 44, -48, 75]", "tst": "[2024,3,25,\"AM\"]", "ted": "[]", "operation": "--coordinate-all"}'
-
-
+      - name: checkout repository
+        uses: actions/checkout@v2
+
+      - name: pip install fireatlas
+        run: |
+          pip install .
+
+      - name: get docker GID and set DOCKER_GID environment variable
+        run: |
+          echo "DOCKER_GID=$(getent group docker | cut -d: -f3)" >> $GITHUB_ENV
+
+      - name: get current user UID and set UID environment variable
+        run: |
+          echo "UID=$(id -u)" >> $GITHUB_ENV
+
+      - name: print DOCKER_GID and UID
+        run: |
+          echo "Docker GID: $DOCKER_GID"
+          echo "User UID: $UID"
+
+      - name: kick off the DPS job
+        uses: Earth-Information-System/fireatlas/.github/actions/run-dps-job-v3@conus-dps
+        with:
+          algo_name: eis-feds-dask-coordinator-v3
+          github_ref: 1.2.3
+          username: gcorradini
+          queue: maap-dps-eis-worker-128gb
+          maap_image_env: ubuntu
+          maap_pgt_secret: ${{ secrets.MAAP_PGT }}
+          json_params: '{"regnm": "BorealNA", "bbox": "[-169, 44, -48, 75]", "tst": "[2024,3,25,\"AM\"]", "ted": "[]", "operation": "--coordinate-all"}'
+
+      - name: send alert on Slack if failure
+        if: always()
+        uses: ravsamhq/notify-slack-action@2.5.0
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: "GH Action Failed - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|View Failed Run>"
+          footer: "<${{github.server_url}}/${{github.repository}}/issues/|Open GitHub Issues>"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK_WEBHOOK_URL }}
diff --git a/.github/workflows/schedule-conus-nrt-v3.yaml b/.github/workflows/schedule-conus-nrt-v3.yaml
@@ -44,4 +44,13 @@ jobs:
         maap_pgt_secret: ${{ secrets.MAAP_PGT }}
         json_params: '{"regnm": "CONUS", "bbox": "[-126.401171875,24.071240929282325,-61.36210937500001,49.40003415463647]", "tst": "[2024,3,25,\"AM\"]", "ted": "[]", "operation": "--coordinate-all"}'
 
-
+    - name: send alert on Slack if failure
+      if: always() 
+      uses: ravsamhq/notify-slack-action@2.5.0
+      with: 
+        status: ${{ job.status }}
+        notify_when: 'failure'
+        notification_title: "GH Action Failed - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|View Failed Run>"
+        footer: "<${{github.server_url}}/${{github.repository}}/issues/|Open GitHub Issues>"
+      env: 
+        SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK_WEBHOOK_URL }}
diff --git a/.github/workflows/schedule-hawaii-nrt-v3.yaml b/.github/workflows/schedule-hawaii-nrt-v3.yaml
@@ -3,43 +3,52 @@ name: hawaii nrt v3
 on:
   workflow_dispatch:
   schedule:
-    - cron:  '40 15,23 * * *'
+    - cron: "40 15,23 * * *"
 
 jobs:
   run_dps_job_v3:
     if: false #contains('["ranchodeluxe", "mccabete"]', github.actor)
     runs-on: ubuntu-latest
     environment: production
     steps:
-    - name: checkout repository
-      uses: actions/checkout@v2
-
-    - name: pip install fireatlas
-      run: |
-        pip install .
-
-    - name: get docker GID and set DOCKER_GID environment variable
-      run: |
-        echo "DOCKER_GID=$(getent group docker | cut -d: -f3)" >> $GITHUB_ENV
-
-    - name: get current user UID and set UID environment variable
-      run: |
-        echo "UID=$(id -u)" >> $GITHUB_ENV
-
-    - name: print DOCKER_GID and UID
-      run: |
-        echo "Docker GID: $DOCKER_GID"
-        echo "User UID: $UID"
-
-    - name: kick off the DPS job
-      uses: Earth-Information-System/fireatlas/.github/actions/run-dps-job-v3@conus-dps
-      with:
-        algo_name: eis-feds-dask-coordinator-v3
-        github_ref: 1.2.3
-        username: gcorradini
-        queue: maap-dps-eis-worker-128gb
-        maap_image_env: ubuntu
-        maap_pgt_secret: ${{ secrets.MAAP_PGT }}
-        json_params: '{"regnm": "Hawaii", "bbox": "[-160.62150215971252, 18.55881382325731, -154.20548653471252, 22.508528755761755]", "tst": "[2024,3,25,\"AM\"]", "ted": "[]", "operation": "--coordinate-all"}'
-
-
+      - name: checkout repository
+        uses: actions/checkout@v2
+
+      - name: pip install fireatlas
+        run: |
+          pip install .
+
+      - name: get docker GID and set DOCKER_GID environment variable
+        run: |
+          echo "DOCKER_GID=$(getent group docker | cut -d: -f3)" >> $GITHUB_ENV
+
+      - name: get current user UID and set UID environment variable
+        run: |
+          echo "UID=$(id -u)" >> $GITHUB_ENV
+
+      - name: print DOCKER_GID and UID
+        run: |
+          echo "Docker GID: $DOCKER_GID"
+          echo "User UID: $UID"
+
+      - name: kick off the DPS job
+        uses: Earth-Information-System/fireatlas/.github/actions/run-dps-job-v3@conus-dps
+        with:
+          algo_name: eis-feds-dask-coordinator-v3
+          github_ref: 1.2.3
+          username: gcorradini
+          queue: maap-dps-eis-worker-128gb
+          maap_image_env: ubuntu
+          maap_pgt_secret: ${{ secrets.MAAP_PGT }}
+          json_params: '{"regnm": "Hawaii", "bbox": "[-160.62150215971252, 18.55881382325731, -154.20548653471252, 22.508528755761755]", "tst": "[2024,3,25,\"AM\"]", "ted": "[]", "operation": "--coordinate-all"}'
+
+      - name: send alert on Slack if failure
+        if: always()
+        uses: ravsamhq/notify-slack-action@2.5.0
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: "GH Action Failed - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|View Failed Run>"
+          footer: "<${{github.server_url}}/${{github.repository}}/issues/|Open GitHub Issues>"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK_WEBHOOK_URL }}
diff --git a/.github/workflows/schedule-nrt-date-update-current-day.yaml b/.github/workflows/schedule-nrt-date-update-current-day.yaml
@@ -3,7 +3,7 @@ name: nrt current day update
 on:
   workflow_dispatch:
   schedule:
-    - cron:  '0 9-23 * * *'
+    - cron: "0 9-23 * * *"
 
 jobs:
   run_dps_job_v3:
@@ -13,35 +13,45 @@ jobs:
     runs-on: ubuntu-latest
     environment: production
     steps:
-    - name: checkout repository
-      uses: actions/checkout@v2
-
-    - name: pip install fireatlas
-      run: |
-        pip install .
-
-    - name: get docker GID and set DOCKER_GID environment variable
-      run: |
-        echo "DOCKER_GID=$(getent group docker | cut -d: -f3)" >> $GITHUB_ENV
-
-    - name: get current user UID and set UID environment variable
-      run: |
-        echo "UID=$(id -u)" >> $GITHUB_ENV
-
-    - name: print DOCKER_GID and UID
-      run: |
-        echo "Docker GID: $DOCKER_GID"
-        echo "User UID: $UID"
-
-    - name: kick off the DPS job
-      uses: Earth-Information-System/fireatlas/.github/actions/run-dps-job-v3@conus-dps
-      with:
-        algo_name: eis-feds-dask-coordinator-v3
-        github_ref: 1.2.3
-        username: gcorradini
-        queue: maap-dps-eis-worker-32gb
-        maap_image_env: ubuntu
-        maap_pgt_secret: ${{ secrets.MAAP_PGT }}
-        # for data update the script expects all params so we can pass nonesense, the only used one is "operation"
-        json_params: '{"regnm": "NA", "bbox": "[0,0,0,0]", "tst": "[]", "ted": "[]", "operation": "--data-update"}'
-
+      - name: checkout repository
+        uses: actions/checkout@v2
+
+      - name: pip install fireatlas
+        run: |
+          pip install .
+
+      - name: get docker GID and set DOCKER_GID environment variable
+        run: |
+          echo "DOCKER_GID=$(getent group docker | cut -d: -f3)" >> $GITHUB_ENV
+
+      - name: get current user UID and set UID environment variable
+        run: |
+          echo "UID=$(id -u)" >> $GITHUB_ENV
+
+      - name: print DOCKER_GID and UID
+        run: |
+          echo "Docker GID: $DOCKER_GID"
+          echo "User UID: $UID"
+
+      - name: kick off the DPS job
+        uses: Earth-Information-System/fireatlas/.github/actions/run-dps-job-v3@conus-dps
+        with:
+          algo_name: eis-feds-dask-coordinator-v3
+          github_ref: 1.2.3
+          username: gcorradini
+          queue: maap-dps-eis-worker-32gb
+          maap_image_env: ubuntu
+          maap_pgt_secret: ${{ secrets.MAAP_PGT }}
+          # for data update the script expects all params so we can pass nonesense, the only used one is "operation"
+          json_params: '{"regnm": "NA", "bbox": "[0,0,0,0]", "tst": "[]", "ted": "[]", "operation": "--data-update"}'
+
+      - name: send alert on Slack if failure
+        if: always()
+        uses: ravsamhq/notify-slack-action@2.5.0
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: "GH Action Failed - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|View Failed Run>"
+          footer: "<${{github.server_url}}/${{github.repository}}/issues/|Open GitHub Issues>"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK_WEBHOOK_URL }}
diff --git a/.github/workflows/schedule-russia-east-nrt-v3.yaml b/.github/workflows/schedule-russia-east-nrt-v3.yaml
@@ -3,7 +3,7 @@ name: russia east nrt v3
 on:
   workflow_dispatch:
   schedule:
-    - cron:  '25 11,23 * * *'
+    - cron: "25 11,23 * * *"
 
 jobs:
   run_dps_job_v3:
@@ -13,37 +13,44 @@ jobs:
     runs-on: ubuntu-latest
     environment: production
     steps:
-    - name: checkout repository
-      uses: actions/checkout@v2
-
-    - name: pip install fireatlas
-      run: |
-        pip install .
-
-    - name: get docker GID and set DOCKER_GID environment variable
-      run: |
-        echo "DOCKER_GID=$(getent group docker | cut -d: -f3)" >> $GITHUB_ENV
-
-    - name: get current user UID and set UID environment variable
-      run: |
-        echo "UID=$(id -u)" >> $GITHUB_ENV
-
-    - name: print DOCKER_GID and UID
-      run: |
-        echo "Docker GID: $DOCKER_GID"
-        echo "User UID: $UID"
-
-    - name: kick off the DPS job
-      uses: Earth-Information-System/fireatlas/.github/actions/run-dps-job-v3@conus-dps
-      with:
-        algo_name: eis-feds-dask-coordinator-v3
-        github_ref: 1.2.3
-        username: gcorradini
-        queue: maap-dps-eis-worker-128gb
-        maap_image_env: ubuntu
-        maap_pgt_secret: ${{ secrets.MAAP_PGT }}
-        json_params: '{"regnm": "RussiaEast", "bbox": "[100.371094, 49.699613, 166.113281, 72.231640]", "tst": "[2024,5,1,\"AM\"]", "ted": "[]", "operation": "--coordinate-all"}'
-
-
-
-
+      - name: checkout repository
+        uses: actions/checkout@v2
+
+      - name: pip install fireatlas
+        run: |
+          pip install .
+
+      - name: get docker GID and set DOCKER_GID environment variable
+        run: |
+          echo "DOCKER_GID=$(getent group docker | cut -d: -f3)" >> $GITHUB_ENV
+
+      - name: get current user UID and set UID environment variable
+        run: |
+          echo "UID=$(id -u)" >> $GITHUB_ENV
+
+      - name: print DOCKER_GID and UID
+        run: |
+          echo "Docker GID: $DOCKER_GID"
+          echo "User UID: $UID"
+
+      - name: kick off the DPS job
+        uses: Earth-Information-System/fireatlas/.github/actions/run-dps-job-v3@conus-dps
+        with:
+          algo_name: eis-feds-dask-coordinator-v3
+          github_ref: 1.2.3
+          username: gcorradini
+          queue: maap-dps-eis-worker-128gb
+          maap_image_env: ubuntu
+          maap_pgt_secret: ${{ secrets.MAAP_PGT }}
+          json_params: '{"regnm": "RussiaEast", "bbox": "[100.371094, 49.699613, 166.113281, 72.231640]", "tst": "[2024,5,1,\"AM\"]", "ted": "[]", "operation": "--coordinate-all"}'
+
+      - name: send alert on Slack if failure
+        if: always()
+        uses: ravsamhq/notify-slack-action@2.5.0
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: "GH Action Failed - <${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}|View Failed Run>"
+          footer: "<${{github.server_url}}/${{github.repository}}/issues/|Open GitHub Issues>"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK_WEBHOOK_URL }}