diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 3ef0bba92168..0313162852be 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -69,6 +69,7 @@ ) from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files from warehouse.rate_limiting.interfaces import RateLimiterException +from warehouse.tuf import targets from warehouse.utils import http, readme from warehouse.utils.project import PROJECT_NAME_RE, validate_project_name from warehouse.utils.security_policy import AuthenticationMethod @@ -1405,6 +1406,9 @@ def file_upload(request): file_data = file_ request.db.add(file_) + # Add the project simple detail and file to TUF Metadata + task = targets.add(request, project, file_) + file_.record_event( tag=EventTag.File.FileAdd, request=request, @@ -1420,6 +1424,7 @@ def file_upload(request): if request.oidc_publisher else None, "project_id": str(project.id), + "tuf": task["data"]["task_id"], }, ) diff --git a/warehouse/manage/views/__init__.py b/warehouse/manage/views/__init__.py index 9d210712a066..b98bd03d223e 100644 --- a/warehouse/manage/views/__init__.py +++ b/warehouse/manage/views/__init__.py @@ -121,7 +121,9 @@ RoleInvitation, RoleInvitationStatus, ) +from warehouse.packaging.utils import render_simple_detail from warehouse.rate_limiting import IRateLimiter +from warehouse.tuf import targets from warehouse.utils.http import is_safe_url from warehouse.utils.paginate import paginate_url_factory from warehouse.utils.project import confirm_project, destroy_docs, remove_project @@ -1835,17 +1837,24 @@ def delete_project_release(self): ) ) + # Delete the project release (simple detail and files) from TUF Metadata + tasks = targets.delete_release(self.request, self.release) + self.release.project.record_event( tag=EventTag.Project.ReleaseRemove, request=self.request, additional={ "submitted_by": self.request.user.username, "canonical_version": self.release.canonical_version, + "tuf": ", ".join([task["data"]["task_id"] for task in tasks]), }, ) self.request.db.delete(self.release) + # Generate new project simple detal and add to TUF Metadata + targets.add(self.request, self.release.project) + self.request.session.flash( self.request._(f"Deleted release {self.release.version!r}"), queue="success" ) @@ -1927,6 +1936,9 @@ def _error(message): ) ) + # Delete the file and project simple detail from TUF metadata + task = targets.delete_file(self.request, self.release.project, release_file) + release_file.record_event( tag=EventTag.File.FileRemove, request=self.request, @@ -1935,6 +1947,7 @@ def _error(message): "canonical_version": self.release.canonical_version, "filename": release_file.filename, "project_id": str(self.release.project.id), + "tuf": task["data"]["task_id"], }, ) @@ -1959,6 +1972,9 @@ def _error(message): self.request.db.delete(release_file) + # Generate new project simple detal and add to TUF Metadata + targets.add(self.request, self.release.project) + self.request.session.flash( f"Deleted file {release_file.filename!r}", queue="success" ) diff --git a/warehouse/packaging/interfaces.py b/warehouse/packaging/interfaces.py index 448620d2842f..ead8d85d1b87 100644 --- a/warehouse/packaging/interfaces.py +++ b/warehouse/packaging/interfaces.py @@ -44,6 +44,11 @@ def get_checksum(path): Return the md5 digest of the file at a given path as a lowercase string. """ + def get_blake2bsum(path): + """ + Return the blake2b digest of the file at a given path as a lowercase string. + """ + def store(path, file_path, *, meta=None): """ Save the file located at file_path to the file storage at the location diff --git a/warehouse/packaging/services.py b/warehouse/packaging/services.py index 66f31ace375a..426dc14ff8fe 100644 --- a/warehouse/packaging/services.py +++ b/warehouse/packaging/services.py @@ -74,6 +74,13 @@ def get_metadata(self, path): def get_checksum(self, path): return hashlib.md5(open(os.path.join(self.base, path), "rb").read()).hexdigest() + def get_blake2bsum(self, path): + content_hasher = hashlib.blake2b(digest_size=256 // 8) + content_hasher.update(open(os.path.join(self.base, path), "rb").read()) + content_hash = content_hasher.hexdigest().lower() + + return content_hash + def store(self, path, file_path, *, meta=None): destination = os.path.join(self.base, path) os.makedirs(os.path.dirname(destination), exist_ok=True) @@ -305,6 +312,20 @@ def get_metadata(self, path): def get_checksum(self, path): raise NotImplementedError + @google.api_core.retry.Retry( + predicate=google.api_core.retry.if_exception_type( + google.api_core.exceptions.ServiceUnavailable + ) + ) + def get_blake2bsum(self, path): + path = self._get_path(path) + blob = self.bucket.blob(path) + content_hasher = hashlib.blake2b(digest_size=256 // 8) + content_hasher.update(blob.download_as_string()) + content_hash = content_hasher.hexdigest().lower() + + return content_hash + @google.api_core.retry.Retry( predicate=google.api_core.retry.if_exception_type( google.api_core.exceptions.ServiceUnavailable diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 4e8226758ed0..ea340e210711 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -121,3 +121,13 @@ def render_simple_detail(project, request, store=False): ) return (content_hash, simple_detail_path, simple_detail_size) + + +def current_simple_details_path(request, project): + storage = request.find_service(ISimpleStorage) + current_hash = storage.get_blake2bsum(f"{project.normalized_name}/index.html") + simple_detail_path = ( + f"{project.normalized_name}/{current_hash}.{project.normalized_name}.html" + ) + + return simple_detail_path diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py index 45d74f3c5061..1002529c4cdb 100644 --- a/warehouse/tuf/__init__.py +++ b/warehouse/tuf/__init__.py @@ -14,4 +14,5 @@ def includeme(config): api_base_url = config.registry.settings["tuf.api.url"] config.add_settings({"tuf.api.task.url": f"{api_base_url}task/"}) + config.add_settings({"tuf.api.targets.url": f"{api_base_url}targets/"}) config.add_settings({"tuf.api.publish.url": f"{api_base_url}targets/publish/"}) diff --git a/warehouse/tuf/targets.py b/warehouse/tuf/targets.py new file mode 100644 index 000000000000..c101e0271854 --- /dev/null +++ b/warehouse/tuf/targets.py @@ -0,0 +1,89 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests + +from pyramid.httpexceptions import HTTPBadGateway + +from warehouse.packaging.models import File +from warehouse.packaging.utils import current_simple_details_path, render_simple_detail + +targets_url = lambda request: request.registry.settings["tuf.api.targets.url"] +publish_url = lambda request: request.registry.settings["tuf.api.publish.url"] + + +def _target_post(path, size, blake2_256_digest): + return { + "path": path, + "info": { + "length": size, + "hashes": {"blake2b-256": blake2_256_digest}, + }, + } + + +def _post_targets(request, targets, publish=True): + payload = { + "targets": targets, + "publish_targets": publish, + } + + rstuf_response = requests.post(targets_url(request), json=payload) + if rstuf_response.status_code != 202: + raise HTTPBadGateway(f"Unexpected TUF Server response: {rstuf_response.text}") + + return rstuf_response.json() + + +def _delete_targets(request, targets, publish=True): + payload = { + "targets": targets, + "publish_targets": publish, + } + + rstuf_response = requests.delete(targets_url(request), json=payload) + if rstuf_response.status_code != 202: + raise HTTPBadGateway(f"Unexpected TUF Server response: {rstuf_response.text}") + + return rstuf_response.json() + + +def add(request, project, file=None): + simple_index = render_simple_detail(project, request, store=True) + targets = [] + targets.append(_target_post(simple_index[1], simple_index[2], simple_index[0])) + if file: + targets.append(_target_post(file.path, file.size, file.blake2_256_digest)) + + task = _post_targets(request, targets) + + return task + + +def delete_file(request, project, file): + # Delete the file and the current simple index from TUF Metadata + current_simple_index = current_simple_details_path(request, project) + targets_to_delete = [file.path, current_simple_index] + task = _delete_targets(request, targets_to_delete) + + # TODO: monitor the TUF tasks for actions + return task + + +def delete_release(request, release): + files = request.db.query(File).filter(File.release_id == release.id).all() + + tasks = [] + for file in files: + tasks.append(delete_file(request, release.project, file)) + + return tasks