From 0e59f72575d5696594969fc022d99b53a20b12ff Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Fri, 9 Jun 2023 09:58:29 +0200 Subject: [PATCH] Manage packages/project and simple details to TUF * Adding packages After adding a package to the Warehouse database, it generates and stores the Simple Index with a request to the RSTUF backend to include the package and its simple index in TUF Metadata. * Removing package or Project Release On PyPI Management, when a user removes a file or a project release it also removes it from TUF metadata and updates the simple details index. Co-authored-by: Lukas Puehringer Signed-off-by: Kairo de Araujo simplify code in warehouse.tuf.targets Signed-off-by: Kairo de Araujo --- .vscode/settings.json | 3 - warehouse/forklift/legacy.py | 5 ++ warehouse/manage/views/__init__.py | 15 +++++ warehouse/packaging/interfaces.py | 5 ++ warehouse/packaging/services.py | 30 ++++++++++ warehouse/packaging/utils.py | 10 ++++ warehouse/tuf/__init__.py | 1 + warehouse/tuf/targets.py | 91 ++++++++++++++++++++++++++++++ 8 files changed, 157 insertions(+), 3 deletions(-) delete mode 100644 .vscode/settings.json create mode 100644 warehouse/tuf/targets.py diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index a7d0fc7b77f6..000000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "esbonio.sphinx.confDir": "" -} \ No newline at end of file diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 3ef0bba92168..dfb35c56b75c 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -69,6 +69,7 @@ ) from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files from warehouse.rate_limiting.interfaces import RateLimiterException +from warehouse.tuf import targets from warehouse.utils import http, readme from warehouse.utils.project import PROJECT_NAME_RE, validate_project_name from warehouse.utils.security_policy import AuthenticationMethod @@ -1405,6 +1406,9 @@ def file_upload(request): file_data = file_ request.db.add(file_) + # Add the project simple detail and file to TUF Metadata + task = targets.add_file(request, project, file_) + file_.record_event( tag=EventTag.File.FileAdd, request=request, @@ -1420,6 +1424,7 @@ def file_upload(request): if request.oidc_publisher else None, "project_id": str(project.id), + "tuf": task["data"]["task_id"], }, ) diff --git a/warehouse/manage/views/__init__.py b/warehouse/manage/views/__init__.py index 9d210712a066..06709a93a6ca 100644 --- a/warehouse/manage/views/__init__.py +++ b/warehouse/manage/views/__init__.py @@ -122,6 +122,7 @@ RoleInvitationStatus, ) from warehouse.rate_limiting import IRateLimiter +from warehouse.tuf import targets from warehouse.utils.http import is_safe_url from warehouse.utils.paginate import paginate_url_factory from warehouse.utils.project import confirm_project, destroy_docs, remove_project @@ -1835,17 +1836,24 @@ def delete_project_release(self): ) ) + # Delete the project release (simple detail and files) from TUF Metadata + tasks = targets.delete_release(self.request, self.release) + self.release.project.record_event( tag=EventTag.Project.ReleaseRemove, request=self.request, additional={ "submitted_by": self.request.user.username, "canonical_version": self.release.canonical_version, + "tuf": ", ".join([task["data"]["task_id"] for task in tasks]), }, ) self.request.db.delete(self.release) + # Generate new project simple detail and add to TUF Metadata + targets.add_file(self.request, self.release.project) + self.request.session.flash( self.request._(f"Deleted release {self.release.version!r}"), queue="success" ) @@ -1927,6 +1935,9 @@ def _error(message): ) ) + # Delete the file and project simple detail from TUF metadata + task = targets.delete_file(self.request, self.release.project, release_file) + release_file.record_event( tag=EventTag.File.FileRemove, request=self.request, @@ -1935,6 +1946,7 @@ def _error(message): "canonical_version": self.release.canonical_version, "filename": release_file.filename, "project_id": str(self.release.project.id), + "tuf": task["data"]["task_id"], }, ) @@ -1959,6 +1971,9 @@ def _error(message): self.request.db.delete(release_file) + # Generate new project simple detail and add to TUF Metadata + targets.add_file(self.request, self.release.project) + self.request.session.flash( f"Deleted file {release_file.filename!r}", queue="success" ) diff --git a/warehouse/packaging/interfaces.py b/warehouse/packaging/interfaces.py index 448620d2842f..ead8d85d1b87 100644 --- a/warehouse/packaging/interfaces.py +++ b/warehouse/packaging/interfaces.py @@ -44,6 +44,11 @@ def get_checksum(path): Return the md5 digest of the file at a given path as a lowercase string. """ + def get_blake2bsum(path): + """ + Return the blake2b digest of the file at a given path as a lowercase string. + """ + def store(path, file_path, *, meta=None): """ Save the file located at file_path to the file storage at the location diff --git a/warehouse/packaging/services.py b/warehouse/packaging/services.py index 66f31ace375a..0e41ce6ce660 100644 --- a/warehouse/packaging/services.py +++ b/warehouse/packaging/services.py @@ -74,6 +74,13 @@ def get_metadata(self, path): def get_checksum(self, path): return hashlib.md5(open(os.path.join(self.base, path), "rb").read()).hexdigest() + def get_blake2bsum(self, path): + content_hasher = hashlib.blake2b(digest_size=256 // 8) + content_hasher.update(open(os.path.join(self.base, path), "rb").read()) + content_hash = content_hasher.hexdigest().lower() + + return content_hash + def store(self, path, file_path, *, meta=None): destination = os.path.join(self.base, path) os.makedirs(os.path.dirname(destination), exist_ok=True) @@ -195,6 +202,9 @@ def create_service(cls, context, request): prefix = request.registry.settings.get("files.prefix") return cls(bucket, prefix=prefix) + def get_blake2bsum(self, path): + raise NotImplementedError + class GenericS3BlobStorage(GenericBlobStorage): def get(self, path): @@ -247,6 +257,9 @@ def create_service(cls, context, request): prefix = request.registry.settings.get("files.prefix") return cls(bucket, prefix=prefix) + def get_blake2bsum(self, path): + raise NotImplementedError + @implementer(IFileStorage) class S3ArchiveFileStorage(GenericS3BlobStorage): @@ -258,6 +271,9 @@ def create_service(cls, context, request): prefix = request.registry.settings.get("archive_files.prefix") return cls(bucket, prefix=prefix) + def get_blake2bsum(self, path): + raise NotImplementedError + @implementer(IDocsStorage) class S3DocsStorage: @@ -305,6 +321,20 @@ def get_metadata(self, path): def get_checksum(self, path): raise NotImplementedError + @google.api_core.retry.Retry( + predicate=google.api_core.retry.if_exception_type( + google.api_core.exceptions.ServiceUnavailable + ) + ) + def get_blake2bsum(self, path): + path = self._get_path(path) + blob = self.bucket.blob(path) + content_hasher = hashlib.blake2b(digest_size=256 // 8) + content_hasher.update(blob.download_as_string()) + content_hash = content_hasher.hexdigest().lower() + + return content_hash + @google.api_core.retry.Retry( predicate=google.api_core.retry.if_exception_type( google.api_core.exceptions.ServiceUnavailable diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 4e8226758ed0..ea340e210711 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -121,3 +121,13 @@ def render_simple_detail(project, request, store=False): ) return (content_hash, simple_detail_path, simple_detail_size) + + +def current_simple_details_path(request, project): + storage = request.find_service(ISimpleStorage) + current_hash = storage.get_blake2bsum(f"{project.normalized_name}/index.html") + simple_detail_path = ( + f"{project.normalized_name}/{current_hash}.{project.normalized_name}.html" + ) + + return simple_detail_path diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py index 45d74f3c5061..1002529c4cdb 100644 --- a/warehouse/tuf/__init__.py +++ b/warehouse/tuf/__init__.py @@ -14,4 +14,5 @@ def includeme(config): api_base_url = config.registry.settings["tuf.api.url"] config.add_settings({"tuf.api.task.url": f"{api_base_url}task/"}) + config.add_settings({"tuf.api.targets.url": f"{api_base_url}targets/"}) config.add_settings({"tuf.api.publish.url": f"{api_base_url}targets/publish/"}) diff --git a/warehouse/tuf/targets.py b/warehouse/tuf/targets.py new file mode 100644 index 000000000000..cb62b9feffd2 --- /dev/null +++ b/warehouse/tuf/targets.py @@ -0,0 +1,91 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests + +from pyramid.httpexceptions import HTTPBadGateway + +from warehouse.packaging.models import File +from warehouse.packaging.utils import current_simple_details_path, render_simple_detail + + +def _payload(targets): + """Helper to create payload for POST or DELETE targets request.""" + return { + "targets": targets, + "publish_targets": True, + } + + +def _payload_targets_part(path, size, digest): + """Helper to create payload part for POST targets request.""" + return { + "path": path, + "info": { + "length": size, + "hashes": {"blake2b-256": digest}, + }, + } + + +def _handle(response): + """Helper to handle http response for POST or DELETE targets request.""" + if response.status_code != 202: + raise HTTPBadGateway(f"Unexpected TUF Server response: {response.text}") + + return response.json() + + +def add_file(request, project, file=None): + """Call RSTUF to add file and new project simple index to TUF targets metadata. + + NOTE: If called without file, only adds new project simple index. This + can be used to re-add project simple index, after deleting a file. + """ + targets = [] + digest, path, size = render_simple_detail(project, request, store=True) + simple_index_part = _payload_targets_part(path, size, digest) + targets.append(simple_index_part) + if file: + file_part = _payload_targets_part(file.path, file.size, file.blake2_256_digest) + targets.append(file_part) + + response = requests.post( + request.registry.settings["tuf.api.targets.url"], json=_payload(targets) + ) + + return _handle(response) + + +def delete_file(request, project, file): + """Call RSTUF to remove file and project simple index from TUF targets metadata. + + NOTE: Simple index needs to be added separately. + """ + index_path = current_simple_details_path(request, project) + targets = [file.path, index_path] + + response = requests.delete( + request.registry.settings["tuf.api.targets.url"], json=_payload(targets) + ) + + return _handle(response) + + +def delete_release(request, release): + files = request.db.query(File).filter(File.release_id == release.id).all() + + tasks = [] + for file in files: + tasks.append(delete_file(request, release.project, file)) + + return tasks