Add workflow for CFEP3 copies (#1047)

* Draft workflow for CFEP3 copies

* Extend PR template

* require SHA256 checks

* use hmac

Co-authored-by: Matthew R. Becker <beckermr@users.noreply.github.com>

* use list of dicts for clarity

* update README

* format

---------

Co-authored-by: Matthew R. Becker <beckermr@users.noreply.github.com>

.github/PULL_REQUEST_TEMPLATE.md

@@ -47,6 +47,12 @@ What will happen when a package is marked broken?
   * [ ] Pinged the relevant feedstock team(s)
   * [ ] Added a small description explaining why access is needed
 
+* [ ] I want to copy an artifact following [CFEP-3](https://github.com/conda-forge/cfep/blob/main/cfep-03.md):
+  * [ ] Pinged the relevant feedstock team(s)
+  * [ ] Added a reference to the original PR
+  * [ ] Posted a link to the conda artifacts
+  * [ ] Posted a link to the build logs
+
 <!--
 For example if you are trying to mark a `foo` conda package as broken.
 

README.md

@@ -63,3 +63,12 @@ Available opt-in resources:
 
 - Travis CI: See `examples/example-travis.yml`
 - [`open-gpu-server`](https://github.com/Quansight/open-gpu-server) (includes GPU CI and long-running builds): See `examples/example-open-gpu-server.yml`. 
+
+## Request a CFEP-3 copy to conda-forge
+
+CFEP-3 specifies the process to add foreign builds to conda-forge. [Read the CFEP](https://github.com/conda-forge/cfep/blob/main/cfep-03.md) for more details.
+This workflow allows users to request a copy once the manual review has been passed.
+
+To do so, please create a new `.yml` file in the `requests` folder. Check `examples/example-cfep-3.yml` for the required metadata.
+
+For provenance and transparency, the PR description must include a link to the original PR and the logs, along with the artifact(s) to be reviewed.

conda_forge_admin_requests/__init__.py

@@ -1,6 +1,6 @@
 import importlib
 import pkgutil
-from . import archive_feedstock, mark_broken, token_reset, access_control
+from . import archive_feedstock, mark_broken, token_reset, access_control, cfep3_copy
 
 actions = {}
 
@@ -21,6 +21,7 @@ def register_actions():
     register_action("token_reset", token_reset)
     register_action("travis", access_control)
     register_action("cirun", access_control)
+    register_action("cfep3_copy", cfep3_copy)
     for pkg in pkgutil.iter_modules():
         if pkg.name.startswith("conda_forge_admin_requests_"):
             spec = importlib.util.find_spec(pkg.name)

conda_forge_admin_requests/cfep3_copy.py

@@ -0,0 +1,102 @@
+"""
+Copy approved artifacts from an external channel to production conda-forge.
+"""
+
+import copy
+import hmac
+import os
+import subprocess
+from typing import Dict, Any
+
+import requests
+
+from .utils import split_label_from_channel, parse_filename
+
+
+def check_one(package: str, sha256: str):
+    if not isinstance(sha256, str) or len(sha256) != 64:
+        raise ValueError(
+            f"Key '{sha256}' must be SHA256 for the artifact (64 hexadecimal characters)"
+        )
+
+    channel_and_maybe_label, subdir, artifact = package.rsplit("/", 2)
+    channel, _ = split_label_from_channel(channel_and_maybe_label)
+    pkg_name, version, _, _ = parse_filename(artifact)
+
+    # Check existence
+    url = (
+        f"https://conda-web.anaconda.org/{channel_and_maybe_label}/{subdir}/{artifact}"
+    )
+    r = requests.head(url)
+    if not r.ok:
+        raise ValueError(f"Package '{package}' at {channel} does not seem to exist")
+
+    # Check SHA256
+    r = requests.get(
+        f"https://api.anaconda.org/dist/{channel}/{pkg_name}/{version}/{subdir}/{artifact}",
+        timeout=10,
+    )
+    r.raise_for_status()
+    api_sha256 = r.json()["sha256"]
+    if not hmac.compare_digest(sha256, api_sha256):
+        raise ValueError(
+            f"User-provided SHA256 {sha256} does not match expected value {api_sha256}"
+        )
+
+
+def check(request: Dict[str, Any]) -> None:
+    packages = request.get("anaconda_org_packages")
+    if not packages or not isinstance(packages[0], dict):
+        raise ValueError(
+            "Must define 'anaconda_org_packages' as a list of dicts with keys "
+            "{'package': channel/subdir/artifact, 'sha256': SHA256}"
+        )
+    for item in packages:
+        if not item.get("package") or not item.get("sha256"):
+            raise ValueError(
+                "Each 'anaconda_org_packages' entry must be a dict with keys "
+                "{'package': channel/subdir/artifact, 'sha256': SHA256}"
+            )
+        check_one(item["package"], item["sha256"])
+
+
+def run(request: Dict[str, Any]) -> Dict[str, Any] | None:
+    if "PROD_BINSTAR_TOKEN" not in os.environ:
+        return copy.deepcopy(request)
+
+    to_label = request.get("to_anaconda_org_label") or ()
+    if to_label:
+        to_label = ("--to-label", to_label)
+    packages_to_try_again = []
+    for item in request["anaconda_org_packages"]:
+        check_one(item["package"], item["sha256"])
+        channel_and_maybe_label, subdir, artifact = item["package"].rsplit("/", 2)
+        channel, label = split_label_from_channel(channel_and_maybe_label)
+        from_label = ("--from-label", label) if label != "main" else ()
+        pkg_name, version, _, _ = parse_filename(artifact)
+        spec = f"{channel}/{pkg_name}/{version}/{subdir}/{artifact}"
+        cmd = [
+            "anaconda",
+            "--token",
+            os.environ["PROD_BINSTAR_TOKEN"],
+            "copy",
+            "--to-owner",
+            "conda-forge",
+            *from_label,
+            *to_label,
+            spec,
+        ]
+        print("Copying", item["package"], "...")
+        p = subprocess.run(cmd)
+        if p.returncode == 0:
+            print("... OK!")
+        else:
+            print("... failed!")
+            packages_to_try_again.append(item)
+
+    if packages_to_try_again:
+        request = copy.deepcopy(request)
+        request["anaconda_org_packages"] = packages_to_try_again
+        return request
+    else:
+        return None

conda_forge_admin_requests/utils.py

@@ -22,3 +22,21 @@ def write_secrets_to_files():
         if token_name in os.environ:
             _write_token(token_fname, os.environ[token_name])
 
+
+def split_label_from_channel(channel: str) -> tuple[str, str]:
+    if "/label/" in channel:
+        return channel.split("/label/", 1)
+    return channel, "main"
+
+
+def parse_filename(filename: str) -> tuple[str, str, str, str]:  
+    if filename.endswith(".tar.bz2"):
+        basename = filename[:-len(".tar.bz2")]
+        extension = "tar.bz2"
+    elif filename.endswith(".conda"):
+        basename = filename[:-len(".conda")]
+        extension = "conda"
+    else:
+        raise ValueError(f"Unknown extension for {filename}")
+    pkg_name, version, build = basename.rsplit("-", 2)
+    return pkg_name, version, build, extension

examples/example-cfep-3.yml

@@ -0,0 +1,6 @@
+action: cfep3_copy
+anaconda_org_packages:
+  # list of {package: channel[/label/<label>]/subdir/filename.extension, sha256: SHA256}
+  - package: jaimergp/label/conda-standalone-24.7.1/osx-arm64/conda-standalone-24.7.1-hce30654_0.conda
+    sha256: cd11f1f0fbe8c203212d002ca4e01795d0b1c886ce47d8b08e6bca3366a7c8e6
+to_anaconda_org_label: main  # optional, default=main