From c5301cb5b09bcae73ebe7ac2917a0809937e0079 Mon Sep 17 00:00:00 2001 From: driazati Date: Thu, 2 Jun 2022 13:51:51 -0700 Subject: [PATCH] [ci] Add guards to pytest_wrapper This should fix #11544 and adds some more logging in case the issue persists. Unfortunately it is difficult to test for real since the case data in that PR is thrown away after Jenkins is done (Jenkins does store test data but it marshals JUnits into its own format) --- Jenkinsfile | 338 +++++++++++++++++++++++++++++++- jenkins/macros.j2 | 12 ++ tests/scripts/git_utils.py | 5 +- tests/scripts/pytest_wrapper.py | 9 +- 4 files changed, 358 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 334448a7ae24..0205a1e7364f 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-06-01T16:34:53.941462 +// Generated at 2022-06-02T14:03:43.284817 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -847,6 +847,14 @@ def shard_run_unittest_GPU_1_of_3() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -899,6 +907,14 @@ def shard_run_unittest_GPU_2_of_3() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -947,6 +963,14 @@ def shard_run_unittest_GPU_3_of_3() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -994,6 +1018,14 @@ def shard_run_integration_CPU_1_of_6() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1040,6 +1072,14 @@ def shard_run_integration_CPU_2_of_6() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1086,6 +1126,14 @@ def shard_run_integration_CPU_3_of_6() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1132,6 +1180,14 @@ def shard_run_integration_CPU_4_of_6() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1178,6 +1234,14 @@ def shard_run_integration_CPU_5_of_6() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1224,6 +1288,14 @@ def shard_run_integration_CPU_6_of_6() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1271,6 +1343,14 @@ def shard_run_python_i386_1_of_5() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1317,6 +1397,14 @@ def shard_run_python_i386_2_of_5() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1362,6 +1450,14 @@ def shard_run_python_i386_3_of_5() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1407,6 +1503,14 @@ def shard_run_python_i386_4_of_5() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1452,6 +1556,14 @@ def shard_run_python_i386_5_of_5() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1498,6 +1610,14 @@ def shard_run_test_Hexagon_1_of_7() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1542,6 +1662,14 @@ def shard_run_test_Hexagon_2_of_7() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1586,6 +1714,14 @@ def shard_run_test_Hexagon_3_of_7() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1630,6 +1766,14 @@ def shard_run_test_Hexagon_4_of_7() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1674,6 +1818,14 @@ def shard_run_test_Hexagon_5_of_7() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1718,6 +1870,14 @@ def shard_run_test_Hexagon_6_of_7() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1762,6 +1922,14 @@ def shard_run_test_Hexagon_7_of_7() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1808,6 +1976,14 @@ def shard_run_integration_aarch64_1_of_4() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1853,6 +2029,14 @@ def shard_run_integration_aarch64_2_of_4() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1898,6 +2082,14 @@ def shard_run_integration_aarch64_3_of_4() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1943,6 +2135,14 @@ def shard_run_integration_aarch64_4_of_4() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -1988,6 +2188,14 @@ def shard_run_topi_GPU_1_of_4() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2032,6 +2240,14 @@ def shard_run_topi_GPU_2_of_4() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2076,6 +2292,14 @@ def shard_run_topi_GPU_3_of_4() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2120,6 +2344,14 @@ def shard_run_topi_GPU_4_of_4() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2165,6 +2397,14 @@ def shard_run_frontend_GPU_1_of_6() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2209,6 +2449,14 @@ def shard_run_frontend_GPU_2_of_6() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2253,6 +2501,14 @@ def shard_run_frontend_GPU_3_of_6() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2297,6 +2553,14 @@ def shard_run_frontend_GPU_4_of_6() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2341,6 +2605,14 @@ def shard_run_frontend_GPU_5_of_6() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2385,6 +2657,14 @@ def shard_run_frontend_GPU_6_of_6() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2435,6 +2715,14 @@ def shard_run_topi_aarch64_1_of_2() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2483,6 +2771,14 @@ def shard_run_topi_aarch64_2_of_2() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2528,6 +2824,14 @@ def shard_run_frontend_aarch64_1_of_2() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2572,6 +2876,14 @@ def shard_run_frontend_aarch64_2_of_2() { }) } } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2742,6 +3054,14 @@ stage('Test') { ) }) } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2787,6 +3107,14 @@ stage('Test') { ) }) } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } @@ -2827,6 +3155,14 @@ stage('Test') { ) }) } finally { + sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) + junit 'build/pytest-results/*.xml' } } diff --git a/jenkins/macros.j2 b/jenkins/macros.j2 index 5a641b73fea8..5d996ce19a55 100644 --- a/jenkins/macros.j2 +++ b/jenkins/macros.j2 @@ -19,6 +19,16 @@ "workspace/exec_${env.EXECUTOR_NUMBER}/{{ folder }}" {%- endmacro -%} +{% macro junit_to_s3() %} +sh( + script: """ + set -eux + aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive + """, + label: 'Upload JUnits to S3', + ) +{% endmacro %} + {% macro sharded_test_step(name, num_shards, node, ws, docker_image, platform, test_method_names) %} {% for shard_index in range(1, num_shards + 1) %} @@ -39,6 +49,7 @@ def {{ method_name }}() { }) } } finally { + {{ junit_to_s3() }} junit 'build/pytest-results/*.xml' } } @@ -86,6 +97,7 @@ def {{ method_name }}() { {{ caller() | indent(width=12) | trim }} }) } finally { + {{ junit_to_s3() | indent(width=4) }} junit 'build/pytest-results/*.xml' } } diff --git a/tests/scripts/git_utils.py b/tests/scripts/git_utils.py index 0e2e85e55243..267756d85905 100644 --- a/tests/scripts/git_utils.py +++ b/tests/scripts/git_utils.py @@ -36,7 +36,7 @@ def post(url: str, body: Optional[Any] = None, auth: Optional[Tuple[str, str]] = req = request.Request(url, headers=headers, method="POST") if auth is not None: auth_str = base64.b64encode(f"{auth[0]}:{auth[1]}".encode()) - req.add_header("Authorization", f"Basic {auth_str}") + req.add_header("Authorization", f"Basic {auth_str.decode()}") if body is None: body = "" @@ -47,8 +47,7 @@ def post(url: str, body: Optional[Any] = None, auth: Optional[Tuple[str, str]] = req.add_header("Content-Length", len(data)) with request.urlopen(req, data) as response: - response = json.loads(response.read()) - return response + return response.read() class GitHubRepo: diff --git a/tests/scripts/pytest_wrapper.py b/tests/scripts/pytest_wrapper.py index a7b6f0dfa766..4c4410bedc9c 100755 --- a/tests/scripts/pytest_wrapper.py +++ b/tests/scripts/pytest_wrapper.py @@ -18,6 +18,7 @@ import argparse import textwrap import junitparser +import traceback from pathlib import Path from typing import List, Optional import os @@ -51,6 +52,10 @@ def failed_test_ids() -> List[str]: for suite in xml: # handle suites for case in suite: + if case.result is None: + logging.warn(f"Incorrectly formatted JUnit found, result was None on {case}") + continue + if len(case.result) > 0 and isinstance(case.result[0], FAILURE_TYPES): node_id = classname_to_file(case.classname) + "::" + case.name failed_node_ids.append(node_id) @@ -112,7 +117,7 @@ def show_failure_help(failed_suites: List[str]) -> None: "If there is no test listed below, the failure likely came from a segmentation " "fault which you can find in the logs above.\n" ) - if len(failed_suites) > 0: + if failed_suites is not None and len(failed_suites) > 0: print("\n".join([f" - {suite}" for suite in failed_suites])) print("") @@ -131,4 +136,4 @@ def show_failure_help(failed_suites: List[str]) -> None: except Exception as e: # This script shouldn't ever introduce failures since it's just there to # add extra information, so ignore any errors - logging.error(str(e)) + logging.exception(e)