Skip to content

Commit

Permalink
[CI] Assert some unittests are not skipped in CI (#12436)
Browse files Browse the repository at this point in the history 
This PR adds a script that does a diff of skipped tests between the latest successful build on the main and the current branch. Then, it posts a comment with the report on the open PR. 

#11670
  • Loading branch information
@gigiblender
gigiblender committed Aug 25, 2022
1 parent cc19cdd commit 56b7c8a
Show file tree
Hide file tree
Showing 3 changed files with 456 additions and 0 deletions.
21 changes: 21 additions & 0 deletions .github/workflows/tests_bot.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@

name: tests-bot
on:
status
jobs:
run-tests-bot:
if: ${{ github.repository == 'apache/tvm' && github.event.state == 'success' && github.event.context == 'tvm-ci/pr-head' }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Comment skipped tests
env:
AWS_ACCESS_KEY_ID: ${{ secrets.CI_RESOURCES_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_RESOURCES_AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-west-2
COMMIT_SHA: ${{ github.event.sha }}
TARGET_URL: ${{ github.event.target_url }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -eux
python tests/scripts/github_skipped_tests_comment.py
179 changes: 179 additions & 0 deletions tests/python/ci/test_ci.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
"""Test various CI scripts and GitHub Actions workflows"""
import shutil
import subprocess
import json
import textwrap
Expand All @@ -33,6 +34,184 @@ def parameterize_named(*values):
return pytest.mark.parametrize(",".join(keys), [tuple(d.values()) for d in values])


# pylint: disable=line-too-long
TEST_DATA_SKIPPED_BOT = {
"found-diff": {
"main_xml_file": "unittest/file1.xml",
"main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
<testsuites>
<testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
<testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
</testcase>
</testsuite>
</testsuites>
""",
"pr_xml_file": "unittest/file2.xml",
"pr_xml_content": """<?xml version="1.0" encoding="utf-8"?>
<testsuites>
<testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
<testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
<skipped message="This test is skipped" type="pytest.skip">
Skipped
</skipped>
</testcase>
<testcase classname="ctypes.tests.python.unittest.test_roofline"
name="test_estimate_peak_bandwidth[cuda]" time="4.679">
<skipped message="This is another skippe test" type="pytest.skip">
Skipped
</skipped>
</testcase>
</testsuite>
</testsuites>
""",
"target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
"s3_prefix": "tvm-jenkins-artifacts-prod",
"jenkins_prefix": "ci.tlcpack.ai",
"common_main_build": """{"build_number": "4115", "state": "success"}""",
"commit_sha": "SHA",
"expected_url": "issues/11594/comments",
"expected_body": """<!---skipped-tests-comment-->\n\nThe list below shows some tests that ran in main SHA but were skipped in the CI build of SHA:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).""",
},
"no-diff": {
"main_xml_file": "unittest/file1.xml",
"main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
<testsuites>
<testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
<testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
<skipped message="This test is skipped" type="pytest.skip">
Skipped
</skipped>
</testcase>
</testsuite>
</testsuites>
""",
"pr_xml_file": "unittest/file2.xml",
"pr_xml_content": """<?xml version="1.0" encoding="utf-8"?>
<testsuites>
<testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
<testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
<skipped message="This test is skipped" type="pytest.skip">
Skipped
</skipped>
</testcase>
</testsuite>
</testsuites>
""",
"target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
"s3_prefix": "tvm-jenkins-artifacts-prod",
"jenkins_prefix": "ci.tlcpack.ai",
"common_main_build": """{"build_number": "4115", "state": "success"}""",
"commit_sha": "SHA",
"expected_url": "issues/11594/comments",
"expected_body": """<!---skipped-tests-comment-->\n\nNo additional skipped tests found in this branch for commit SHA.""",
},
"unable-to-run": {
"main_xml_file": "unittest/file1.xml",
"main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
<testsuites>
</testsuites>
""",
"pr_xml_file": "unittest/file2.xml",
"pr_xml_content": """<?xml version="1.0" encoding="utf-8"?>
<testsuites>
</testsuites>
""",
"target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
"s3_prefix": "tvm-jenkins-artifacts-prod",
"jenkins_prefix": "ci.tlcpack.ai",
"common_main_build": """{"build_number": "4115", "state": "failed"}""",
"commit_sha": "SHA",
"expected_url": "issues/11594/comments",
"expected_body": """<!---skipped-tests-comment-->\n\nUnable to run tests bot because main failed to pass CI at SHA.""",
},
}
# pylint: enable=line-too-long


@tvm.testing.skip_if_wheel_test
@pytest.mark.parametrize(
[
"main_xml_file",
"main_xml_content",
"pr_xml_file",
"pr_xml_content",
"target_url",
"s3_prefix",
"jenkins_prefix",
"common_main_build",
"commit_sha",
"expected_url",
"expected_body",
],
[tuple(d.values()) for d in TEST_DATA_SKIPPED_BOT.values()],
ids=TEST_DATA_SKIPPED_BOT.keys(),
)
# pylint: enable=line-too-long
def test_skipped_tests_comment(
tmpdir_factory,
main_xml_file,
main_xml_content,
pr_xml_file,
pr_xml_content,
target_url,
s3_prefix,
jenkins_prefix,
common_main_build,
commit_sha,
expected_url,
expected_body,
):
"""
Test that a comment with a link to the docs is successfully left on PRs
"""
skipped_tests_script = REPO_ROOT / "tests" / "scripts" / "github_skipped_tests_comment.py"

def write_xml_file(root_dir, xml_file, xml_content):
shutil.rmtree(root_dir, ignore_errors=True)
file = root_dir / xml_file
file.parent.mkdir(parents=True)
with open(file, "w") as f:
f.write(textwrap.dedent(xml_content))

git = TempGit(tmpdir_factory.mktemp("tmp_git_dir"))
git.run("init")
git.run("checkout", "-b", "main")
git.run("remote", "add", "origin", "https://github.com/apache/tvm.git")

pr_test_report_dir = Path(git.cwd) / "pr-reports"
write_xml_file(pr_test_report_dir, pr_xml_file, pr_xml_content)
main_test_report_dir = Path(git.cwd) / "main-reports"
write_xml_file(main_test_report_dir, main_xml_file, main_xml_content)

proc = subprocess.run(
[
str(skipped_tests_script),
"--dry-run",
f"--s3-prefix={s3_prefix}",
f"--jenkins-prefix={jenkins_prefix}",
f"--common-main-build={common_main_build}",
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env={"TARGET_URL": target_url, "COMMIT_SHA": commit_sha},
encoding="utf-8",
cwd=git.cwd,
check=False,
)
if proc.returncode != 0:
raise RuntimeError(f"Process failed:\nstdout:\n{proc.stdout}\n\nstderr:\n{proc.stderr}")

assert f"Dry run, would have posted {expected_url} with data {expected_body}." in proc.stderr


@tvm.testing.skip_if_wheel_test
@pytest.mark.parametrize(
"target_url,base_url,commit_sha,expected_url,expected_body",
Expand Down
Loading

0 comments on commit 56b7c8a

Please sign in to comment.