Manage packages, simple index to TUF metadata

* Adding packages

After adding a package to the Warehouse database, it generates and
stores the Simple Index with a request to the RSTUF backend to
include the package and its simple index in TUF Metadata.

* Removing package or Project Release

On PyPI Management, when a user removes a file or a project release
it also removes it from TUF metadata and updates the simple details index.

Signed-off-by: Kairo de Araujo <kdearaujo@vmware.com>

warehouse/forklift/legacy.py

@@ -69,6 +69,7 @@
 )
 from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files
 from warehouse.rate_limiting.interfaces import RateLimiterException
+from warehouse.tuf import targets
 from warehouse.utils import http, readme
 from warehouse.utils.project import PROJECT_NAME_RE, validate_project_name
 from warehouse.utils.security_policy import AuthenticationMethod
@@ -1405,6 +1406,9 @@ def file_upload(request):
         file_data = file_
         request.db.add(file_)
 
+        # Add the project simple detail and file to TUF Metadata
+        task = targets.add(request, project, file_)
+
         file_.record_event(
             tag=EventTag.File.FileAdd,
             request=request,
@@ -1420,6 +1424,7 @@ def file_upload(request):
                 if request.oidc_publisher
                 else None,
                 "project_id": str(project.id),
+                "tuf": task["data"]["task_id"],
             },
         )
 

warehouse/manage/views/__init__.py

@@ -121,7 +121,9 @@
     RoleInvitation,
     RoleInvitationStatus,
 )
+from warehouse.packaging.utils import render_simple_detail
 from warehouse.rate_limiting import IRateLimiter
+from warehouse.tuf import targets
 from warehouse.utils.http import is_safe_url
 from warehouse.utils.paginate import paginate_url_factory
 from warehouse.utils.project import confirm_project, destroy_docs, remove_project
@@ -1835,17 +1837,24 @@ def delete_project_release(self):
             )
         )
 
+        # Delete the project release (simple detail and files) from TUF Metadata
+        tasks = targets.delete_release(self.request, self.release)
+
         self.release.project.record_event(
             tag=EventTag.Project.ReleaseRemove,
             request=self.request,
             additional={
                 "submitted_by": self.request.user.username,
                 "canonical_version": self.release.canonical_version,
+                "tuf": ", ".join([task["data"]["task_id"] for task in tasks]),
             },
         )
 
         self.request.db.delete(self.release)
 
+        # Generate new project simple detal and add to TUF Metadata
+        targets.add(self.request, self.release.project)
+
         self.request.session.flash(
             self.request._(f"Deleted release {self.release.version!r}"), queue="success"
         )
@@ -1927,6 +1936,9 @@ def _error(message):
             )
         )
 
+        # Delete the file and project simple detail from TUF metadata
+        task = targets.delete_file(self.request, self.release.project, release_file)
+
         release_file.record_event(
             tag=EventTag.File.FileRemove,
             request=self.request,
@@ -1935,6 +1947,7 @@ def _error(message):
                 "canonical_version": self.release.canonical_version,
                 "filename": release_file.filename,
                 "project_id": str(self.release.project.id),
+                "tuf": task["data"]["task_id"],
             },
         )
 
@@ -1959,6 +1972,9 @@ def _error(message):
 
         self.request.db.delete(release_file)
 
+        # Generate new project simple detal and add to TUF Metadata
+        targets.add(self.request, self.release.project)
+
         self.request.session.flash(
             f"Deleted file {release_file.filename!r}", queue="success"
         )

warehouse/packaging/interfaces.py

@@ -44,6 +44,11 @@ def get_checksum(path):
         Return the md5 digest of the file at a given path as a lowercase string.
         """
 
+    def get_blake2bsum(path):
+        """
+        Return the blake2b digest of the file at a given path as a lowercase string.
+        """
+
     def store(path, file_path, *, meta=None):
         """
         Save the file located at file_path to the file storage at the location

warehouse/packaging/services.py

@@ -74,6 +74,13 @@ def get_metadata(self, path):
     def get_checksum(self, path):
         return hashlib.md5(open(os.path.join(self.base, path), "rb").read()).hexdigest()
 
+    def get_blake2bsum(self, path):
+        content_hasher = hashlib.blake2b(digest_size=256 // 8)
+        content_hasher.update(open(os.path.join(self.base, path), "rb").read())
+        content_hash = content_hasher.hexdigest().lower()
+
+        return content_hash
+
     def store(self, path, file_path, *, meta=None):
         destination = os.path.join(self.base, path)
         os.makedirs(os.path.dirname(destination), exist_ok=True)
@@ -305,6 +312,20 @@ def get_metadata(self, path):
     def get_checksum(self, path):
         raise NotImplementedError
 
+    @google.api_core.retry.Retry(
+        predicate=google.api_core.retry.if_exception_type(
+            google.api_core.exceptions.ServiceUnavailable
+        )
+    )
+    def get_blake2bsum(self, path):
+        path = self._get_path(path)
+        blob = self.bucket.blob(path)
+        content_hasher = hashlib.blake2b(digest_size=256 // 8)
+        content_hasher.update(blob.download_as_string())
+        content_hash = content_hasher.hexdigest().lower()
+
+        return content_hash
+
     @google.api_core.retry.Retry(
         predicate=google.api_core.retry.if_exception_type(
             google.api_core.exceptions.ServiceUnavailable

warehouse/packaging/utils.py

@@ -121,3 +121,13 @@ def render_simple_detail(project, request, store=False):
             )
 
     return (content_hash, simple_detail_path, simple_detail_size)
+
+
+def current_simple_details_path(request, project):
+    storage = request.find_service(ISimpleStorage)
+    current_hash = storage.get_blake2bsum(f"{project.normalized_name}/index.html")
+    simple_detail_path = (
+        f"{project.normalized_name}/{current_hash}.{project.normalized_name}.html"
+    )
+
+    return simple_detail_path

warehouse/tuf/__init__.py

@@ -14,4 +14,5 @@
 def includeme(config):
     api_base_url = config.registry.settings["tuf.api.url"]
     config.add_settings({"tuf.api.task.url": f"{api_base_url}task/"})
+    config.add_settings({"tuf.api.targets.url": f"{api_base_url}targets/"})
     config.add_settings({"tuf.api.publish.url": f"{api_base_url}targets/publish/"})

warehouse/tuf/targets.py

@@ -0,0 +1,89 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import requests
+
+from pyramid.httpexceptions import HTTPBadGateway
+
+from warehouse.packaging.models import File
+from warehouse.packaging.utils import current_simple_details_path, render_simple_detail
+
+targets_url = lambda request: request.registry.settings["tuf.api.targets.url"]
+publish_url = lambda request: request.registry.settings["tuf.api.publish.url"]
+
+
+def _target_post(path, size, blake2_256_digest):
+    return {
+        "path": path,
+        "info": {
+            "length": size,
+            "hashes": {"blake2b-256": blake2_256_digest},
+        },
+    }
+
+
+def _post_targets(request, targets, publish=True):
+    payload = {
+        "targets": targets,
+        "publish_targets": publish,
+    }
+
+    rstuf_response = requests.post(targets_url(request), json=payload)
+    if rstuf_response.status_code != 202:
+        raise HTTPBadGateway(f"Unexpected TUF Server response: {rstuf_response.text}")
+
+    return rstuf_response.json()
+
+
+def _delete_targets(request, targets, publish=True):
+    payload = {
+        "targets": targets,
+        "publish_targets": publish,
+    }
+
+    rstuf_response = requests.delete(targets_url(request), json=payload)
+    if rstuf_response.status_code != 202:
+        raise HTTPBadGateway(f"Unexpected TUF Server response: {rstuf_response.text}")
+
+    return rstuf_response.json()
+
+
+def add(request, project, file=None):
+    simple_index = render_simple_detail(project, request, store=True)
+    targets = []
+    targets.append(_target_post(simple_index[1], simple_index[2], simple_index[0]))
+    if file:
+        targets.append(_target_post(file.path, file.size, file.blake2_256_digest))
+
+    task = _post_targets(request, targets)
+
+    return task
+
+
+def delete_file(request, project, file):
+    # Delete the file and the current simple index from TUF Metadata
+    current_simple_index = current_simple_details_path(request, project)
+    targets_to_delete = [file.path, current_simple_index]
+    task = _delete_targets(request, targets_to_delete)
+
+    # TODO: monitor the TUF tasks for actions
+    return task
+
+
+def delete_release(request, release):
+    files = request.db.query(File).filter(File.release_id == release.id).all()
+
+    tasks = []
+    for file in files:
+        tasks.append(delete_file(request, release.project, file))
+
+    return tasks