Simpler controls for mercurial local clone

bot/code_review_bot/__init__.py

@@ -185,11 +185,10 @@ def hash(self):
         #  format `obj-x86_64-pc-linux-gnu`
         file_content = None
         if "/obj-" not in self.path:
-            local_repository = settings.runtime.get("mercurial_repository")
-            if local_repository:
+            if settings.mercurial_cache_checkout:
                 logger.debug("Using the local repository to build issue's hash")
                 try:
-                    with open(local_repository / self.path) as f:
+                    with (settings.mercurial_cache_checkout / self.path).open() as f:
                         file_content = f.read()
                 except (FileNotFoundError, IsADirectoryError):
                     logger.warning(

bot/code_review_bot/cli.py

@@ -103,14 +103,9 @@ def main():
         taskcluster.secrets["APP_CHANNEL"],
         taskcluster.secrets["ALLOWED_PATHS"],
         taskcluster.secrets["repositories"],
+        args.mercurial_repository,
     )
 
-    # Store some CLI parameters in the global settings
-    settings.runtime.update(
-        {
-            "mercurial_repository": args.mercurial_repository,
-        }
-    )
     # Setup statistics
     influx_conf = taskcluster.secrets.get("influxdb")
     if influx_conf:

bot/code_review_bot/config.py

@@ -11,7 +11,7 @@
 import os
 import shutil
 import tempfile
-from contextlib import contextmanager
+from pathlib import Path
 
 import pkg_resources
 import structlog
@@ -49,18 +49,21 @@ def __init__(self):
         self.try_group_id = None
         self.autoland_group_id = None
         self.mozilla_central_group_id = None
-        self.hgmo_cache = tempfile.mkdtemp(suffix="hgmo")
         self.repositories = []
         self.decision_env_prefixes = []
-        # Runtime settings
-        self.runtime = {}
+
+        # Cache to store file-by-file from HGMO Rest API
+        self.hgmo_cache = tempfile.mkdtemp(suffix="hgmo")
+
+        # Cache to store whole repositories
+        self.mercurial_cache = None
 
         # Always cleanup at the end of the execution
         atexit.register(self.cleanup)
         # caching the versions of the app
         self.version = pkg_resources.require("code-review-bot")[0].version
 
-    def setup(self, app_channel, allowed_paths, repositories):
+    def setup(self, app_channel, allowed_paths, repositories, mercurial_cache=None):
         # Detect source from env
         if "TRY_TASK_ID" in os.environ and "TRY_TASK_GROUP_ID" in os.environ:
             self.try_task_id = os.environ["TRY_TASK_ID"]
@@ -112,6 +115,13 @@ def build_conf(nb, repo):
             repo.decision_env_prefix for repo in self.repositories
         ]
 
+        # Store mercurial cache path
+        if mercurial_cache is not None:
+            self.mercurial_cache = Path(mercurial_cache)
+            assert (
+                self.mercurial_cache.exists()
+            ), f"Mercurial cache does not exist {self.mercurial_cache}"
+
     def __getattr__(self, key):
         if key not in self.config:
             raise AttributeError
@@ -124,21 +134,29 @@ def on_production(self):
         """
         return self.app_channel == "production" and self.taskcluster.local is False
 
-    def is_allowed_path(self, path):
+    @property
+    def mercurial_cache_checkout(self):
         """
-        Is this path allowed for reporting ?
+        When local mercurial cache is enabled, path to the checkout
         """
-        return any([fnmatch.fnmatch(path, rule) for rule in self.allowed_paths])
+        if self.mercurial_cache is None:
+            return
+        return self.mercurial_cache / "checkout"
+
+    @property
+    def mercurial_cache_sharebase(self):
+        """
+        When local mercurial cache is enabled, path to the shared folder for robust checkout
+        """
+        if self.mercurial_cache is None:
+            return
+        return self.mercurial_cache / "shared"
 
-    @contextmanager
-    def override_runtime_setting(self, key, value):
+    def is_allowed_path(self, path):
         """
-        Overrides a runtime setting, then restores the default value
+        Is this path allowed for reporting ?
         """
-        copy = {**self.runtime}
-        self.runtime[key] = value
-        yield
-        self.runtime = copy
+        return any([fnmatch.fnmatch(path, rule) for rule in self.allowed_paths])
 
     def cleanup(self):
         shutil.rmtree(self.hgmo_cache)

bot/code_review_bot/mercurial.py

@@ -6,9 +6,6 @@
 import fcntl
 import os
 import time
-from contextlib import contextmanager
-from pathlib import Path
-from tempfile import TemporaryDirectory
 
 import hglib
 import structlog
@@ -62,7 +59,7 @@ def _log_process(output, name):
     return out
 
 
-def robust_checkout(repo_url, checkout_dir, sharebase_dir, branch):
+def robust_checkout(repo_url, checkout_dir, sharebase_dir, branch="tip"):
     cmd = hglib.util.cmdbuilder(
         "robustcheckout",
         repo_url,
@@ -72,18 +69,3 @@ def robust_checkout(repo_url, checkout_dir, sharebase_dir, branch):
         branch=branch,
     )
     hg_run(cmd)
-
-
-@contextmanager
-def clone_repository(repo_url, branch="tip"):
-    """
-    Clones a repository to a temporary directory using robustcheckout.
-    """
-    with TemporaryDirectory() as temp_path:
-        temp_path = Path(temp_path)
-        sharebase_dir = (temp_path / "shared").absolute()
-        sharebase_dir.mkdir()
-        # Do not create the checkout folder or Mercurial will complain about a missing .hg file
-        checkout_dir = (temp_path / "checkout").absolute()
-        robust_checkout(repo_url, str(checkout_dir), str(sharebase_dir), branch)
-        yield checkout_dir

bot/code_review_bot/workflow.py

@@ -3,7 +3,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from contextlib import nullcontext
 from datetime import datetime, timedelta
 from itertools import groupby
 
@@ -14,7 +13,7 @@
 from code_review_bot import Level, stats
 from code_review_bot.backend import BackendAPI
 from code_review_bot.config import REPO_AUTOLAND, REPO_MOZILLA_CENTRAL, settings
-from code_review_bot.mercurial import clone_repository
+from code_review_bot.mercurial import robust_checkout
 from code_review_bot.report.debug import DebugReporter
 from code_review_bot.revisions import Revision
 from code_review_bot.tasks.base import AnalysisTask, BaseTask, NoticeTask
@@ -203,28 +202,27 @@ def _build_tasks(tasks):
             logger.info("No issues for that revision")
             return
 
-        runtime_repo = settings.runtime.get("mercurial_repository")
-        context_manager = nullcontext(runtime_repo)
-        # Do always clone the repository on production to speed up reading issues
-        if runtime_repo is None and settings.taskcluster.task_id != "local instance":
+        # Clone the repo locally when configured
+        # On production this should use a Taskcluster cache
+        if settings.mercurial_cache:
             logger.info(
                 "Cloning revision to build issues",
                 repo=revision.head_repository,
                 changeset=revision.head_changeset,
             )
-            context_manager = clone_repository(
-                repo_url=revision.head_repository, branch=revision.head_changeset
+            robust_checkout(
+                repo_url=revision.head_repository,
+                branch=revision.head_changeset,
+                checkout_dir=settings.mercurial_cache_checkout,
+                sharebase_dir=settings.mercurial_cache_sharebase,
             )
 
-        with context_manager as repo_path:
-            # Override runtime settings with the cloned repository
-            with settings.override_runtime_setting("mercurial_repository", repo_path):
-                # Publish issues in the backend
-                self.backend_api.publish_issues(
-                    issues,
-                    revision,
-                    bulk=BULK_ISSUE_CHUNKS,
-                )
+        # Publish issues in the backend
+        self.backend_api.publish_issues(
+            issues,
+            revision,
+            bulk=BULK_ISSUE_CHUNKS,
+        )
 
     def publish(self, revision, issues, task_failures, notices, reviewers):
         """

bot/tests/test_mercurial.py

@@ -3,8 +3,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from pathlib import PosixPath
-
 from code_review_bot import mercurial
 
 
@@ -42,28 +40,24 @@ def test_hg_run(monkeypatch):
     assert popen_mock.stderr.content == "An error occurred"
 
 
-def test_clone_repository_context_manager(monkeypatch):
+def test_robustcheckout(monkeypatch):
     popen_mock = PopenMock()
     monkeypatch.setattr("hglib.util.popen", popen_mock)
 
-    with mercurial.clone_repository(
-        "https://hg.repo/", branch="default"
-    ) as repo_checkout:
-        assert isinstance(repo_checkout, PosixPath)
-        assert str(repo_checkout.absolute()).startswith("/tmp/")
-        assert repo_checkout.stem == "checkout"
-        parent_folder = repo_checkout.parent.absolute()
-        assert parent_folder.exists()
-        assert str(parent_folder) != "/tmp"
+    mercurial.robust_checkout(
+        "https://hg.repo/",
+        branch="default",
+        checkout_dir="/tmp/checkout",
+        sharebase_dir="/tmp/shared",
+    )
 
     assert popen_mock.command == [
         "hg",
         "robustcheckout",
         b"--purge",
-        f"--sharebase={parent_folder}/shared".encode(),
+        b"--sharebase=/tmp/shared",
         b"--branch=default",
         b"--",
         "https://hg.repo/",
-        str(repo_checkout),
+        "/tmp/checkout",
     ]
-    assert not parent_folder.exists()