[CI] Locate build for base main commit instead of using the latest bu…

…ild on main
apache · Aug 24, 2022 · 1d88434 · 1d88434
1 parent 0ab966c
commit 1d88434
Show file tree

Hide file tree

Showing 2 changed files with 92 additions and 24 deletions.
diff --git a/tests/python/ci/test_ci.py b/tests/python/ci/test_ci.py
@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 """Test various CI scripts and GitHub Actions workflows"""
-import os
 import shutil
 import subprocess
 import json
@@ -39,7 +38,7 @@ def parameterize_named(*values):
 @pytest.mark.parametrize(
     # pylint: disable=line-too-long
     "main_xml_file,main_xml_content,pr_xml_file,pr_xml_content,target_url,s3_prefix,"
-    "jenkins_prefix,commit_sha,expected_url,expected_body",
+    "jenkins_prefix,common_main_build,commit_sha,expected_url,expected_body",
     [
         (
             "unittest/file1.xml",
@@ -76,9 +75,10 @@ def parameterize_named(*values):
             "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
             "tvm-jenkins-artifacts-prod",
             "ci.tlcpack.ai",
+            """{"build_number": "4115", "state": "success"}""",
             "SHA",
             "issues/11594/comments",
-            """<!---skipped-tests-comment-->\n\nThe list below shows some tests that ran in main but were skipped in the CI build of SHA:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).""",
+            """<!---skipped-tests-comment-->\n\nThe list below shows some tests that ran in main SHA but were skipped in the CI build of SHA:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).""",
         ),
         (
             "unittest/file1.xml",
@@ -112,10 +112,30 @@ def parameterize_named(*values):
             "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
             "tvm-jenkins-artifacts-prod",
             "ci.tlcpack.ai",
+            """{"build_number": "4115", "state": "success"}""",
             "SHA",
             "issues/11594/comments",
             """<!---skipped-tests-comment-->\n\nNo additional skipped tests found in this branch for commit SHA.""",
         ),
+        (
+            "unittest/file1.xml",
+            """<?xml version="1.0" encoding="utf-8"?>
+                    <testsuites>
+                    </testsuites>
+                    """,
+            "unittest/file2.xml",
+            """<?xml version="1.0" encoding="utf-8"?>
+                    <testsuites>
+                    </testsuites>
+                    """,
+            "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
+            "tvm-jenkins-artifacts-prod",
+            "ci.tlcpack.ai",
+            """{"build_number": "4115", "state": "failed"}""",
+            "SHA",
+            "issues/11594/comments",
+            """<!---skipped-tests-comment-->\n\nUnable to run tests bot because main failed to pass CI at SHA.""",
+        ),
     ],
 )
 # pylint: enable=line-too-long
@@ -128,6 +148,7 @@ def test_skipped_tests_comment(
     target_url,
     s3_prefix,
     jenkins_prefix,
+    common_main_build,
     commit_sha,
     expected_url,
     expected_body,
@@ -139,8 +160,8 @@ def test_skipped_tests_comment(
 
     def write_xml_file(root_dir, xml_file, xml_content):
         shutil.rmtree(root_dir, ignore_errors=True)
-        file = f"""{root_dir}/{xml_file}"""
-        os.makedirs(os.path.dirname(file))
+        file = root_dir / xml_file
+        file.parent.mkdir(parents=True)
         with open(file, "w") as f:
             f.write(textwrap.dedent(xml_content))
 
@@ -149,9 +170,9 @@ def write_xml_file(root_dir, xml_file, xml_content):
     git.run("checkout", "-b", "main")
     git.run("remote", "add", "origin", "https://github.com/apache/tvm.git")
 
-    pr_test_report_dir = git.cwd + "/pr-reports"
+    pr_test_report_dir = Path(git.cwd) / "pr-reports"
     write_xml_file(pr_test_report_dir, pr_xml_file, pr_xml_content)
-    main_test_report_dir = git.cwd + "/main-reports"
+    main_test_report_dir = Path(git.cwd) / "main-reports"
     write_xml_file(main_test_report_dir, main_xml_file, main_xml_content)
 
     proc = subprocess.run(
@@ -160,6 +181,7 @@ def write_xml_file(root_dir, xml_file, xml_content):
             "--dry-run",
             f"--s3-prefix={s3_prefix}",
             f"--jenkins-prefix={jenkins_prefix}",
+            f"--common-main-build={common_main_build}",
         ],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,

diff --git a/tests/scripts/github_skipped_tests_comment.py b/tests/scripts/github_skipped_tests_comment.py
@@ -15,7 +15,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
+import json
 import os
 import logging
 import argparse
@@ -36,25 +36,50 @@
 MAIN_TEST_REPORT_DIR = "main-reports"
 
 
-def retrieve_test_report(s3_url, target_dir):
-    command = f"aws s3 cp {s3_url} {target_dir} --recursive"
+def run_subprocess(command):
     logging.info(f"Running command {command}")
     proc = subprocess.run(command, shell=True, stdout=subprocess.PIPE, encoding="utf-8")
     if proc.returncode != 0:
         raise RuntimeError(f"Command failed {command}:\nstdout:\n{proc.stdout}")
+    return proc
+
+
+def retrieve_test_report(s3_url, target_dir):
+    command = f"aws s3 cp {s3_url} {target_dir} --recursive"
+    run_subprocess(command)
+
+
+def get_common_commit_sha():
+    command = "git merge-base origin/main HEAD"
+    proc = run_subprocess(command)
+    return proc.stdout.strip()
 
 
-def retrieve_test_reports(pr_number, build_number, s3_prefix, jenkins_prefix):
+def get_main_jenkins_build_number(github, common_commit):
+    json = github.get(f"commits/{common_commit}/status")
+    for status in reversed(json["statuses"]):
+        if status["context"] != "tvm-ci/branch":
+            continue
+        state = status["state"]
+        target_url = str(status["target_url"])
+        build_number = (
+            target_url[target_url.find("job/main") : len(target_url)]
+            .strip("job/main/")
+            .strip("/display/redirect")
+        )
+        assert build_number.isdigit()
+        return {"build_number": build_number, "state": state}
+    raise RuntimeError(f"Failed to find main build number for commit {common_commit}")
+
+
+def retrieve_test_reports(common_main_build, pr_number, build_number, s3_prefix):
     cur_build_s3_link = (
         f"s3://{s3_prefix}/tvm/PR-{str(pr_number)}/{str(build_number)}/pytest-results"
     )
     retrieve_test_report(cur_build_s3_link, PR_TEST_REPORT_DIR)
 
-    latest_main_build = requests.get(
-        f"https://{jenkins_prefix}/job/tvm/job/main/lastSuccessfulBuild/buildNumber"
-    ).text
-    latest_build_s3_link = f"s3://{s3_prefix}/tvm/main/{latest_main_build}/pytest-results"
-    retrieve_test_report(latest_build_s3_link, MAIN_TEST_REPORT_DIR)
+    common_build_s3_link = f"s3://{s3_prefix}/tvm/main/{common_main_build}/pytest-results"
+    retrieve_test_report(common_build_s3_link, MAIN_TEST_REPORT_DIR)
 
 
 def get_pr_and_build_numbers(target_url):
@@ -87,13 +112,24 @@ def to_node_name(dir_name: str):
     return dir_name.replace("_", ": ", 1)
 
 
-def build_comment(skipped_list, pr_number, build_number, commit_sha, jenkins_prefix):
+def build_comment(
+    common_commit_sha,
+    common_main_build,
+    skipped_list,
+    pr_number,
+    build_number,
+    commit_sha,
+    jenkins_prefix,
+):
+    if common_main_build["state"] != "success":
+        return f"{SKIPPED_TESTS_COMMENT_MARKER}Unable to run tests bot because main failed to pass CI at {common_commit_sha}."
+
     if len(skipped_list) == 0:
         return f"{SKIPPED_TESTS_COMMENT_MARKER}No additional skipped tests found in this branch for commit {commit_sha}."
 
     text = (
-        f"{SKIPPED_TESTS_COMMENT_MARKER}The list below shows some tests that ran in main but were skipped in the "
-        f"CI build of {commit_sha}:\n"
+        f"{SKIPPED_TESTS_COMMENT_MARKER}The list below shows some tests that ran in main {common_commit_sha} but were "
+        f"skipped in the CI build of {commit_sha}:\n"
         f"```\n"
     )
     for skip in skipped_list:
@@ -132,6 +168,7 @@ def search_for_docs_comment(comments):
     parser.add_argument("--remote", default="origin", help="ssh remote to parse")
     parser.add_argument("--s3-prefix", default="tvm-jenkins-artifacts-prod")
     parser.add_argument("--jenkins-prefix", default="ci.tlcpack.ai")
+    parser.add_argument("--common-main-build")
     parser.add_argument(
         "--dry-run",
         action="store_true",
@@ -150,12 +187,19 @@ def search_for_docs_comment(comments):
     commit_sha = os.environ["COMMIT_SHA"]
 
     if not args.dry_run:
+        github = GitHubRepo(token=os.environ["GITHUB_TOKEN"], user=user, repo=repo)
+        common_commit_sha = get_common_commit_sha()
+        common_main_build = get_main_jenkins_build_number(github, common_commit_sha)
         retrieve_test_reports(
+            common_main_build=common_main_build["build_number"],
             pr_number=pr_and_build["pr_number"],
             build_number=pr_and_build["build_number"],
             s3_prefix=args.s3_prefix,
-            jenkins_prefix=args.jenkins_prefix,
         )
+    else:
+        assert args.common_main_build is not None
+        common_main_build = json.loads(args.common_main_build)
+        common_commit_sha = os.environ["COMMIT_SHA"]
 
     main_tests = build_test_set(MAIN_TEST_REPORT_DIR)
     build_tests = build_test_set(PR_TEST_REPORT_DIR)
@@ -176,6 +220,8 @@ def search_for_docs_comment(comments):
         logging.info("No skipped tests found.")
 
     body = build_comment(
+        common_commit_sha,
+        common_main_build,
         skipped_list,
         pr_and_build["pr_number"],
         pr_and_build["build_number"],
@@ -184,8 +230,6 @@ def search_for_docs_comment(comments):
     )
     url = f'issues/{pr_and_build["pr_number"]}/comments'
     if not args.dry_run:
-        github = GitHubRepo(token=os.environ["GITHUB_TOKEN"], user=user, repo=repo)
-
         # For now, only comment for PRs open by driazati, gigiblender and areusch.
         get_pr_url = f'pulls/{pr_and_build["pr_number"]}'
         pull_request_body = github.get(get_pr_url)
@@ -199,8 +243,10 @@ def search_for_docs_comment(comments):
 
         if comment is not None:
             comment_url = comment["url"]
-            comment_id = comment_url[comment_url.find("comments/"): len(comment_url)].strip("comments/")
-            github.patch(f'issues/comments/{comment_id}', {"body": body})
+            comment_id = comment_url[comment_url.find("comments/") : len(comment_url)].strip(
+                "comments/"
+            )
+            github.patch(f"issues/comments/{comment_id}", {"body": body})
         else:
             github.post(url, {"body": body})
     else: