[TT-1262] dump pg on failure (#14029)

* bump CTF * bump bump * go mod * bump bump * bump to 1.34.0 * dump Postgres db on failure and upload as artifacts * test dump in CI * remove test failing on demand, fix test-summary action input name * use tagged CTF, save sql dump also if flag is set
smartcontractkit · Aug 6, 2024 · d1d0f44 · d1d0f44
1 parent d963b0a
commit d1d0f44
Show file tree

Hide file tree

Showing 19 changed files with 138 additions and 84 deletions.
diff --git a/.github/workflows/client-compatibility-tests.yml b/.github/workflows/client-compatibility-tests.yml
@@ -641,6 +641,7 @@ jobs:
           artifacts_name: ${{ env.TEST_LOG_NAME }}
           artifacts_location: |
             ./integration-tests/smoke/logs/
+            ./integration-tests/smoke/db_dumps/      
             /tmp/gotest.log
           publish_check_name: ${{ matrix.evm_node.product }}-${{ matrix.evm_node.eth_implementation }}
           token: ${{ secrets.GITHUB_TOKEN }}

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -359,6 +359,7 @@ jobs:
           artifacts_name: ${{ matrix.product.name }}-test-logs
           artifacts_location: |
             ./integration-tests/smoke/logs/
+            ./integration-tests/smoke/db_dumps/
             /tmp/gotest.log
           publish_check_name: ${{ matrix.product.name }}
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -472,6 +473,7 @@ jobs:
           artifacts_name: ${{ matrix.product.name }}-test-logs
           artifacts_location: |
             ./integration-tests/smoke/logs/
+            ./integration-tests/smoke/db_dumps/
             /tmp/gotest.log
           publish_check_name: ${{ matrix.product.name }}
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -709,6 +711,7 @@ jobs:
           artifacts_name: ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}-test-logs
           artifacts_location: |
             ./integration-tests/smoke/logs/
+            ./integration-tests/smoke/db_dumps/
             /tmp/gotest.log
           publish_check_name: ${{ matrix.product.name }}
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -777,7 +780,7 @@ jobs:
         if: always()
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
         with:
-          test_directory: ./integration-tests/smoke/
+          test_directories: ./integration-tests/smoke/
 
   ### Used to check the required checks box when the matrix completes
   eth-smoke-tests:
@@ -978,7 +981,7 @@ jobs:
           DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary"
           DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
           DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
-          
+
       - name: Upload Coverage Data
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         timeout-minutes: 2

diff --git a/.github/workflows/live-testnet-tests.yml b/.github/workflows/live-testnet-tests.yml
@@ -302,7 +302,7 @@ jobs:
         if: always()
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
         with:
-          test_directory: "./"
+          test_directories: "./"
 
   bsc-testnet-smoke-tests:
     environment: integration

diff --git a/.github/workflows/live-vrf-tests.yml b/.github/workflows/live-vrf-tests.yml
@@ -120,7 +120,7 @@ jobs:
     needs: [build-chainlink, build-tests]
     strategy:
       fail-fast: false
-      matrix: 
+      matrix:
         network: ${{fromJson(needs.build-tests.outputs.matrix)}}
     name: Smoke Tests on ${{ matrix.network }}
     runs-on: ubuntu-latest
@@ -190,4 +190,4 @@ jobs:
         if: always()
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25
         with:
-          test_directory: "./"
+          test_directories: "./"
diff --git a/.github/workflows/on-demand-keeper-smoke-tests.yml b/.github/workflows/on-demand-keeper-smoke-tests.yml
@@ -149,6 +149,7 @@ jobs:
           artifacts_name: ${{ matrix.product.name }}-test-logs
           artifacts_location: |
             ./integration-tests/smoke/logs/
+            ./integration-tests/smoke/db_dumps/
             /tmp/gotest.log
           publish_check_name: ${{ matrix.product.name }}
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -286,4 +287,4 @@ jobs:
             go test -run=NonExistentTest ./smoke/... || echo "ignore expected test failure"
           go_mod_path: ./integration-tests/go.mod
           cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }}
-          cache_restore_only: "false"
+          cache_restore_only: "false"
diff --git a/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml b/.github/workflows/on-demand-vrfv2-eth2-clients-test.yml
@@ -5,12 +5,12 @@ on:
       base64Config:
         description: base64-ed config
         required: true
-        type: string 
+        type: string
       test_secrets_override_key:
         description: 'Key to run tests with custom test secrets'
         required: false
-        type: string 
-        
+        type: string
+
 jobs:
   vrfv2_smoke_test:
     name: VRFV2 Smoke Test with custom EL client client
@@ -24,11 +24,11 @@ jobs:
     env:
       TEST_LOG_LEVEL: debug
       REF_NAME: ${{ github.head_ref || github.ref_name }}
-    steps:      
+    steps:
       - name: Checkout code
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
         with:
-          fetch-depth: 0    
+          fetch-depth: 0
       - name: Mask base64 config
         run: |
           BASE64_CONFIG_OVERRIDE=$(jq -r '.inputs.base64Config' $GITHUB_EVENT_PATH)
@@ -37,7 +37,7 @@ jobs:
       - name: Parse base64 config
         uses: ./.github/actions/setup-parse-base64-config
         with:
-          base64Config: ${{ env.BASE64_CONFIG_OVERRIDE }}  
+          base64Config: ${{ env.BASE64_CONFIG_OVERRIDE }}
       - name: Send details to Step Summary
         shell: bash
         run: |
@@ -48,7 +48,7 @@ jobs:
           echo "### Networks on which test was run" >>$GITHUB_STEP_SUMMARY
           echo "\`${{ env.NETWORKS }}\`" >>$GITHUB_STEP_SUMMARY
           echo "### Execution client used" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.ETH2_EL_CLIENT }}\`" >>$GITHUB_STEP_SUMMARY          
+          echo "\`${{ env.ETH2_EL_CLIENT }}\`" >>$GITHUB_STEP_SUMMARY
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@2967f2287bd3f3ddbac7b476e9568993df01796e # v2.3.27
         with:
@@ -59,17 +59,19 @@ jobs:
           cl_image_tag: ${{ env.CHAINLINK_VERSION }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_name: vrf-test-logs
-          artifacts_location: ./integration-tests/smoke/logs/
+          artifacts_location: |
+            ./integration-tests/smoke/logs/
+            ./integration-tests/smoke/db_dumps/
           token: ${{ secrets.GITHUB_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
           should_cleanup: false
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}          
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
           DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }}
           DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
           DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
           DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
           DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary"
           DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+          DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
diff --git a/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml b/.github/workflows/on-demand-vrfv2plus-eth2-clients-test.yml
@@ -5,11 +5,11 @@ on:
       base64Config:
         description: base64-ed config
         required: true
-        type: string  
+        type: string
       test_secrets_override_key:
         description: 'Key to run tests with custom test secrets'
         required: false
-        type: string              
+        type: string
 
 jobs:
   vrfv2plus_smoke_test:
@@ -24,7 +24,7 @@ jobs:
     env:
       TEST_LOG_LEVEL: debug
       REF_NAME: ${{ github.head_ref || github.ref_name }}
-    steps:            
+    steps:
       - name: Checkout code
         uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
         with:
@@ -48,7 +48,7 @@ jobs:
           echo "### Networks on which test was run" >>$GITHUB_STEP_SUMMARY
           echo "\`${{ env.NETWORKS }}\`" >>$GITHUB_STEP_SUMMARY
           echo "### Execution client used" >>$GITHUB_STEP_SUMMARY
-          echo "\`${{ env.ETH2_EL_CLIENT }}\`" >>$GITHUB_STEP_SUMMARY              
+          echo "\`${{ env.ETH2_EL_CLIENT }}\`" >>$GITHUB_STEP_SUMMARY
       - name: Run Tests
         uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@2967f2287bd3f3ddbac7b476e9568993df01796e # v2.3.27
         with:
@@ -59,17 +59,19 @@ jobs:
           cl_image_tag: ${{ env.CHAINLINK_VERSION }}
           aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
           artifacts_name: vrfplus-test-logs
-          artifacts_location: ./integration-tests/smoke/logs/
+          artifacts_location: |
+            ./integration-tests/smoke/logs/
+            ./integration-tests/smoke/db_dumps/
           token: ${{ secrets.GITHUB_TOKEN }}
           go_mod_path: ./integration-tests/go.mod
           should_cleanup: false
           QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
-          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}          
+          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
           QA_KUBECONFIG: ""
           DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }}
           DEFAULT_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
           DEFAULT_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push
           DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
           DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary"
           DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs"
-          DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
+          DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}