diff --git a/Makefile b/Makefile index b6231b5d7f05..dfa6e7de5c77 100644 --- a/Makefile +++ b/Makefile @@ -117,6 +117,14 @@ reindex: .state/docker-build-base shell: .state/docker-build-base docker compose run --rm web python -m warehouse shell +tufinit: + docker compose run --rm web psql -h db -d postgres -U postgres -c "CREATE DATABASE rstuf ENCODING 'UTF8'" + docker compose restart rstuf-worker01 rstuf-worker02 + docker compose run --rm web rstuf admin ceremony -b -u -f /opt/warehouse/src/dev/rstuf-bootstrap-payload.json --upload-server http://rstuf-api + +tufimport: + docker-compose run --rm web python -m warehouse tuf dev import-all + dbshell: .state/docker-build-base docker compose run --rm web psql -h db -d warehouse -U postgres @@ -131,4 +139,4 @@ purge: stop clean stop: docker compose stop -.PHONY: default build serve initdb shell dbshell tests dev-docs user-docs deps clean purge debug stop compile-pot runmigrations +.PHONY: default build serve initdb shell dbshell tests dev-docs user-docs deps clean purge debug stop compile-pot runmigrations tufinit tufimport diff --git a/dev/environment b/dev/environment index 94c05b6c2812..f6914ffb6883 100644 --- a/dev/environment +++ b/dev/environment @@ -68,3 +68,10 @@ OIDC_AUDIENCE=pypi # Default to the reCAPTCHA testing keys from https://developers.google.com/recaptcha/docs/faq RECAPTCHA_SITE_KEY=6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI RECAPTCHA_SECRET_KEY=6LeIxAcTAAAAAGG-vFI1TnRWxMZNFuojJ4WifJWe + +TUF_ENABLED=true +TUF_METADATA_URL="http://files:9001/metadata/" +TUF_API_URL="http://rstuf-api/api/v1/" +TUF_DATABASE_URL="postgresql://postgres@db/rstuf" +TUF_ROOT_SECRET="an insecure private key password" +TUF_ONLINE_SECRET="an insecure private key password" diff --git a/dev/rstuf-bootstrap-payload.json b/dev/rstuf-bootstrap-payload.json new file mode 100644 index 000000000000..1d9c0d042de8 --- /dev/null +++ b/dev/rstuf-bootstrap-payload.json @@ -0,0 +1,90 @@ +{ + "settings": { + "expiration": { + "root": 365, + "targets": 365, + "snapshot": 1, + "timestamp": 1, + "bins": 1 + }, + "services": { + "number_of_delegated_bins": 256, + "targets_base_url": "\"http://127.0.0.1:9001/simple/\"/", + "targets_online_key": true + } + }, + "metadata": { + "root": { + "signatures": [ + { + "keyid": "a0cb8f1d00f8c7455e92272e01f551fc96c38d3b6bd201d7d3bdc08b3a418d1d", + "sig": "6fe3f661a40677df1ff5fac724cf3a47c826224be5ff9e1099cb76f826bac64722fa5e8120ad7eb032565a75a561d69255985b9de4ec25bb115710e8d3602d0b" + }, + { + "keyid": "d5a3a5b1d77c59675fb830a558b7925a6b3e4da2e888af7372094984fbe37e9e", + "sig": "12485c76a748feed1ffdef59c24ba3258e56a20304207ae42138fff2c8c7314a14fb8f0beb7adfe85e78aebfc75200bac233a18a02d8c79ff06813f3900ff50e" + } + ], + "signed": { + "_type": "root", + "version": 1, + "spec_version": "1.0.30", + "expires": "2024-06-11T16:40:02Z", + "consistent_snapshot": true, + "keys": { + "a0cb8f1d00f8c7455e92272e01f551fc96c38d3b6bd201d7d3bdc08b3a418d1d": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyval": { + "public": "ac5cd92ec491fea3f0b4c8a04af3fb957b5fc8965a79379131cfa4581905739f" + }, + "name": "root key 1" + }, + "d5a3a5b1d77c59675fb830a558b7925a6b3e4da2e888af7372094984fbe37e9e": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyval": { + "public": "6d112f8658d1d8f42b17a263641bf7bd8940c97f25f9ea83d3aa609ec5fe9a91" + }, + "name": "root key 2" + }, + "64b5a379908148215a6bc1c9c66aa595fc87037555a054c4dddae5fc96d75bc2": { + "keytype": "ed25519", + "scheme": "ed25519", + "keyval": { + "public": "41df147e582fb6c14445da4db011b7d7d03824ea7b64aef5bb3aa8a57269b327" + }, + "name": "online key v1" + } + }, + "roles": { + "root": { + "keyids": [ + "a0cb8f1d00f8c7455e92272e01f551fc96c38d3b6bd201d7d3bdc08b3a418d1d", + "d5a3a5b1d77c59675fb830a558b7925a6b3e4da2e888af7372094984fbe37e9e" + ], + "threshold": 1 + }, + "timestamp": { + "keyids": [ + "64b5a379908148215a6bc1c9c66aa595fc87037555a054c4dddae5fc96d75bc2" + ], + "threshold": 1 + }, + "snapshot": { + "keyids": [ + "64b5a379908148215a6bc1c9c66aa595fc87037555a054c4dddae5fc96d75bc2" + ], + "threshold": 1 + }, + "targets": { + "keyids": [ + "64b5a379908148215a6bc1c9c66aa595fc87037555a054c4dddae5fc96d75bc2" + ], + "threshold": 1 + } + } + } + } + } +} \ No newline at end of file diff --git a/dev/tufkeys/online/online b/dev/tufkeys/online/online new file mode 100644 index 000000000000..e77865087230 --- /dev/null +++ b/dev/tufkeys/online/online @@ -0,0 +1 @@ +efd5e924987f59b3700a4188b83ae4dd@@@@100000@@@@c3cf5853b7cd2250cb72a1c0b4141c7367acddd3bcb6e96f6ec560f5f50e1c9e@@@@2480ed147201dd75b2d8f20b4a56534c@@@@58e914497e885078dbd8b5e6d10c1c435d8b1f83b53c35c8aa27fd5c703bcd3cc4b7ec8af0ef1d8444f5cad2a54093831944ac425777133f91c98df71018932f3fae77533f8f489f2bdbc63c0faddf2a00a63da37bd292f3ce7b35e86b7ddd90d0f2d92eaa9a264fda9eeb85f714b6745a9ff5ea3e3cd466d94557b0dce8fde3503a12ffd4ddfa0beaf5e509ce3514d071dc26af385dcd23f239711efbc86b2f736027f7940f9a3a786cb3a329158e5a0487ce50ee3a5ed4a032d6e556181a9ffb26c20800d0b4cd0b2149afee333986c0722101603f1d144d4df14a493376d516e7cbd54aa070f96c630672e8b2e8c89c7fe6d44d6cae87188289bdc8361635c613a06cd81f4b6f630938f003aa224c1ff3b31ec01c78fe5e85fcefcfe2d3beb33213b5e244c21b73841d46f47b9360e7bd6c75b8af6aa398dffdae5bce50795d57d37a3df0b3088392c1d050a46302 \ No newline at end of file diff --git a/dev/tufkeys/online/online.pub b/dev/tufkeys/online/online.pub new file mode 100755 index 000000000000..4c6ee59bb1c5 --- /dev/null +++ b/dev/tufkeys/online/online.pub @@ -0,0 +1 @@ +{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "41df147e582fb6c14445da4db011b7d7d03824ea7b64aef5bb3aa8a57269b327"}} \ No newline at end of file diff --git a/dev/tufkeys/root/root1 b/dev/tufkeys/root/root1 new file mode 100644 index 000000000000..430db3754b70 --- /dev/null +++ b/dev/tufkeys/root/root1 @@ -0,0 +1 @@ +0b81cc88ce39650626eba6a9b0420dad@@@@100000@@@@875fed5a914d59843f5280260145405c7599026e915efca75c7773b49b33b2e6@@@@bfb692965db58b72b5f1e9bceaeccc37@@@@81d62e800c1a22b8a82b7c795a2b77990494577dc098755b78d02eac175d3b96c5a5d4cc6e08c17410069118840c32443b7be43c3663963bfdf58f813cdcfd33a29770392ad35c05df7edbd51aaa0e6a6122b752cc876628633d10078fcf494818cf62abe41591ce9543abc4d0ea7a8e731097f1b97e2915358c015d61dadbdefe9a9be8138bdd87d89b952e89817951b60fee1089dcdadd4937e638571f47bd1258276803163797174dee1e963c731f40affaf35804117a5ee555767ea05bb165a3d4751a40d6e0dfc519049e2346dfcd09f851637328883251027ef3bb1566a1c4b1b95f127a636d0499a28fa52eade9e53d9560879ed56346582045750403978f575a4d753abd6e00e46c264f073e858693210ae04df8ce0520ca6fac64a668f61000e9a4b16c29cbcaf96dd35d5489bf1e36cbef3ee2927dffaf3ba822532653542a4516c3cacb3db4e2bca825cf \ No newline at end of file diff --git a/dev/tufkeys/root/root1.pub b/dev/tufkeys/root/root1.pub new file mode 100755 index 000000000000..d92c86c56945 --- /dev/null +++ b/dev/tufkeys/root/root1.pub @@ -0,0 +1 @@ +{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "ac5cd92ec491fea3f0b4c8a04af3fb957b5fc8965a79379131cfa4581905739f"}} \ No newline at end of file diff --git a/dev/tufkeys/root/root2 b/dev/tufkeys/root/root2 new file mode 100644 index 000000000000..7cac09b699c8 --- /dev/null +++ b/dev/tufkeys/root/root2 @@ -0,0 +1 @@ +370fd8314f62bf0743f150e9d9ec1883@@@@100000@@@@b89bb6032fa26cba87e51649a6e83df8a6ff92b81e2616a2af2f3794ce96aee2@@@@67c0d2cde784d5d354817ef42321d75d@@@@2558be0a0ed1b3db89be41669bb6657dcb85e05b1e41a2d57319fb715779cc230940ea88614cf96bf0f0f07a1f8726a780bf2003013c28c36956f597238d64502c3d063e8cf0d953f883f41f7787bcc7c233ca9e6c08e0fc0fdb988f99de80e456dc80d86f087a7535d5e6bed7db11feb4af247c04e01c3b7c0e658cfd6fb170e6370240cc7b0f9cfe0d15723122ae70c56d10487cc19b4dacb047ac8194cc1435a2e687bdcf20f2b4aca227581b3939b0c8aa712d8237bc8dbd977d64c4548359c75d0dad452e0a5517ce02da0db1bf5a077782e7997126f789e3ac93e3cf57feb08b9bf988ba8ff8b42dbf09bc5bbafbd8366d3ab1ebc0fe03899a48abdc55f324d96ef3a70265a728d5ce06f9d35c7b93ad399902be82f87f82f1eecbb1d4666031d515a2f9f14d6160560a9b505a444af8a79358e46609cfa339df8adacca9ee7cf7643b71da535032ae79934e7d \ No newline at end of file diff --git a/dev/tufkeys/root/root2.pub b/dev/tufkeys/root/root2.pub new file mode 100755 index 000000000000..4aa673c5302b --- /dev/null +++ b/dev/tufkeys/root/root2.pub @@ -0,0 +1 @@ +{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "6d112f8658d1d8f42b17a263641bf7bd8940c97f25f9ea83d3aa609ec5fe9a91"}} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index d6cf02ccc83b..e76d801165b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,8 @@ volumes: policies: vault: caches: + rstuf-worker-data: + rstuf-metadata: services: vault: @@ -109,6 +111,8 @@ services: # Included to support linters during development - ./gunicorn-prod.conf.py:/opt/warehouse/src/gunicorn-prod.conf.py:z - ./gunicorn-uploads.conf.py:/opt/warehouse/src/gunicorn-uploads.conf.py:z + - rstuf-metadata:/var/opt/warehouse/metadata + - ./dev/tufkeys:/opt/warehouse/src/dev/tufkeys:z web: image: warehouse:docker-compose @@ -138,6 +142,7 @@ services: - packages-archive:/var/opt/warehouse/packages-archive - sponsorlogos:/var/opt/warehouse/sponsorlogos - simple:/var/opt/warehouse/simple + - rstuf-metadata:/var/opt/warehouse/metadata ports: - "9001:9001" @@ -156,6 +161,68 @@ services: ARCHIVE_FILES_BACKEND: "warehouse.packaging.services.LocalArchiveFileStorage path=/var/opt/warehouse/packages-archive/ url=http://files:9001/packages-archive/{path}" SIMPLE_BACKEND: "warehouse.packaging.services.LocalSimpleStorage path=/var/opt/warehouse/simple/ url=http://files:9001/simple/{path}" + rstuf-worker01: + image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:latest + volumes: + - rstuf-worker-data:/data + - ./dev/rstuf-workers-supervisor.conf:/opt/repository-service-tuf/supervisor.conf:z + - rstuf-metadata:/var/opt/repository-service-tuf/storage + - ./dev/tufkeys/online:/var/opt/repository-service-tuf/keystorage + environment: + - RSTUF_STORAGE_BACKEND=LocalStorage + - RSTUF_LOCAL_STORAGE_BACKEND_PATH=/var/opt/repository-service-tuf/storage + - RSTUF_KEYVAULT_BACKEND=LocalKeyVault + - RSTUF_LOCAL_KEYVAULT_PATH=/var/opt/repository-service-tuf/keystorage + - RSTUF_LOCAL_KEYVAULT_KEYS=online,an insecure private key password + - RSTUF_BROKER_SERVER=redis://redis/1 + - RSTUF_REDIS_SERVER=redis://redis + - RSTUF_REDIS_SERVER_DB_RESULT=1 + - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2 + - RSTUF_SQL_SERVER=postgresql://postgres@db:5432/rstuf + healthcheck: + test: "exit 0" + restart: always + tty: true + depends_on: + db: + condition: service_healthy + + rstuf-worker02: + image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:latest + volumes: + - rstuf-worker-data:/data + - ./dev/rstuf-workers-supervisor.conf:/opt/repository-service-tuf/supervisor.conf:z + - rstuf-metadata:/var/opt/repository-service-tuf/storage + - ./dev/tufkeys/online:/var/opt/repository-service-tuf/keystorage + environment: + - RSTUF_STORAGE_BACKEND=LocalStorage + - RSTUF_LOCAL_STORAGE_BACKEND_PATH=/var/opt/repository-service-tuf/storage + - RSTUF_KEYVAULT_BACKEND=LocalKeyVault + - RSTUF_LOCAL_KEYVAULT_PATH=/var/opt/repository-service-tuf/keystorage + - RSTUF_LOCAL_KEYVAULT_KEYS=online,an insecure private key password + - RSTUF_BROKER_SERVER=redis://redis/1 + - RSTUF_REDIS_SERVER=redis://redis + - RSTUF_REDIS_SERVER_DB_RESULT=1 + - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2 + - RSTUF_SQL_SERVER=postgresql://postgres@db:5432/rstuf + healthcheck: + test: "exit 0" + restart: always + tty: true + depends_on: + db: + condition: service_healthy + + rstuf-api: + image: ghcr.io/repository-service-tuf/repository-service-tuf-api:latest + ports: + - 8001:80 + environment: + - RSTUF_BROKER_SERVER=redis://redis/1 + - RSTUF_REDIS_SERVER=redis://redis + - RSTUF_REDIS_SERVER_DB_RESULT=1 + - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2 + static: build: context: . diff --git a/requirements/dev.txt b/requirements/dev.txt index c3ecdfbb0676..ff39594ae826 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,3 +2,8 @@ asyncudp>=0.7 hupper>=1.9 pip-tools>=1.0 pyramid_debugtoolbar>=2.5 +repository-service-tuf==0.3.0a1 +securesystemslib +dynaconf +rich-click +commonmark \ No newline at end of file diff --git a/warehouse/cli/tuf.py b/warehouse/cli/tuf.py new file mode 100644 index 000000000000..906e73a151f6 --- /dev/null +++ b/warehouse/cli/tuf.py @@ -0,0 +1,206 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import hashlib +import json +import os +import tempfile +import time + +from datetime import datetime +from typing import Any, Dict, List + +import click +import requests + +from sqlalchemy import MetaData, Table, create_engine +from sqlalchemy.dialects.sqlite import insert +from sqlalchemy.engine import Connection +from tuf.api.metadata import Delegations, Metadata, Targets + +from warehouse.cli import warehouse +from warehouse.packaging.tasks import ( + generate_projects_simple_detail as _generate_projects_simple_detail, +) +from warehouse.packaging.utils import render_simple_detail + + +@warehouse.group() # pragma: no-branch +def tuf(): + """ + Manage Warehouse's TUF state. + """ + + +@tuf.group() +def dev(): + """ + TUF Development purposes commands + """ + + +def _fetch_succinct_roles(config): + from tuf.ngclient import Updater # noqa + + metadata_base_url = f"{config.registry.settings['tuf.metadata.url']}" + with tempfile.TemporaryDirectory() as temp_dir: + metadata_dir = f"{temp_dir}/metadata" + os.makedirs(metadata_dir) + target_dir = f"{temp_dir}/targets" + + root_response = requests.get(f"{metadata_base_url}1.root.json") + if root_response.status_code != 200: + raise click.ClickException( + "Failed to retrieve Root Metadata: {root_response.text}" + ) + + with open(f"{metadata_dir}/root.json", "w") as root_file: + json.dump(root_response.json(), root_file) + + updater = Updater( + metadata_dir=metadata_dir, + metadata_base_url=metadata_base_url, + target_dir=target_dir, + ) + + _timestamp = updater._load_timestamp() + _snapshot = updater._load_snapshot() + targets = updater._load_targets("targets", "root") + + if targets.signed.delegations is None: + raise click.ClickException("Failed to get Targets Delegations") + + targets_delegations: Delegations = targets.signed.delegations + if targets_delegations.succinct_roles is None: + raise click.ClickException("Failed to get Targets succinct roles") + succinct_roles = targets_delegations.succinct_roles + + return succinct_roles + + +def _rstuf_target_files(path, size, blake2b_hash, target_role_id): + return { + "path": path, + "info": { + "length": size, + "hashes": {"blake2b-256": blake2b_hash}, + }, + "published": False, + "action": "ADD", + "last_update": datetime.now(), + "targets_role": target_role_id, + } + + +def _get_target_role_id(db, table, succinct_roles, path): + return db.execute( + table.select().where( + table.c.rolename == succinct_roles.get_role_for_target(path) + ) + ).one()[0] + + +@dev.command() +@click.pass_obj +def import_all(config): + """ + Collect every PyPI package and add as targets. + + Collect the "paths" for every PyPI package and Project index and add as + targets. These are packages already in existence, so we'll add all directly + to RSTUF database as a import process + """ + + from warehouse.db import Session + from warehouse.packaging.models import File, Project + + succinct_roles = _fetch_succinct_roles(config) + db = Session(bind=config.registry["sqlalchemy.engine"]) + engine = create_engine(config.registry.settings["tuf.database.url"]) + db_metadata = MetaData() + db_client = engine.connect() + rstuf_target_files = Table("rstuf_target_files", db_metadata, autoload_with=engine) + rstuf_target_roles = Table("rstuf_target_roles", db_metadata, autoload_with=engine) + click.echo(f"Getting all Projects and generating package indexes") + rstuf_db_data_indexes: List[Dict[str, Any]] = [] + all_projects = db.query(Project).all() + request = config.task(_generate_projects_simple_detail).get_request() + request.db = db + click.echo(f"Generating simple detail index and adding to RSTUF tables") + for project in all_projects: + try: + package_index = render_simple_detail(project, request, store=True) + except OSError: + continue + + rstuf_db_data_indexes.append( + _rstuf_target_files( + package_index[1], + package_index[2], + package_index[0], + _get_target_role_id( + db_client, rstuf_target_roles, succinct_roles, package_index[1] + ), + ) + ) + + # FIXME: why do we have duplicated for simple detail indexes? + db_client.execute( + insert(rstuf_target_files) + .values(rstuf_db_data_indexes) + .on_conflict_do_nothing() + ) + db.commit() + click.echo(f"Getting all Packages") + files = db.query(File).all() + click.echo(f"Importing all packages to RSTUF tables") + rstuf_db_data_packages: List[Dict[str, Any]] = [] + for file in files: + rstuf_db_data_packages.append( + _rstuf_target_files( + file.path, + file.size, + file.blake2_256_digest, + _get_target_role_id( + db_client, rstuf_target_roles, succinct_roles, file.path + ), + ) + ) + db_client.execute(rstuf_target_files.insert(), rstuf_db_data_packages) + db.commit() + + # Publish all packages and indexes + click.echo(f"Publishing all packages and indexes in TUF metadata") + publish_targets_url = f"{config.registry.settings['tuf.api.publish.url']}" + publish_all_targets = requests.post(publish_targets_url) + + # Monitoring publish all packages task + if publish_all_targets.status_code != 202: + raise click.ClickException(f"Error: {publish_all_targets.text}") + + task_id = publish_all_targets.json()["data"]["task_id"] + click.echo(f"Publishing packages in TUF metadata. Task id: {task_id}") + click.echo("Waiting packages and indexes to be published.", nl=False) + task_url = f"{config.registry.settings['tuf.api.task.url']}?task_id={task_id}" + while True: + task_status = requests.get(task_url) + if task_status.status_code != 200: + raise click.ClickException(f"Error: {task_status.text}") + state = task_status.json()["data"]["state"] + if state == "STARTED" or state == "PENDING": + click.echo(".", nl=False) + time.sleep(5) + elif state == "SUCCESS": + click.echo("\nAll packages and indexes published.") + break + else: + raise click.ClickException(f"Error: unexpected state {state}") diff --git a/warehouse/config.py b/warehouse/config.py index d81e0df16512..2d5a18638960 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -241,6 +241,13 @@ def configure(settings=None): coercer=int, default=100, ) + maybe_set(settings, "tuf.enabled", "TUF_ENABLED") + maybe_set(settings, "tuf.database.url", "TUF_DATABASE_URL") + maybe_set(settings, "tuf.metadata.url", "TUF_METADATA_URL") + maybe_set(settings, "tuf.api.url", "TUF_API_URL") + maybe_set(settings, "tuf.root1.secret", "TUF_ROOT_SECRET") + maybe_set(settings, "tuf.root2.secret", "TUF_ROOT_SECRET") + maybe_set(settings, "tuf.online.secret", "TUF_ROOT_SECRET") maybe_set_compound(settings, "billing", "backend", "BILLING_BACKEND") maybe_set_compound(settings, "files", "backend", "FILES_BACKEND") maybe_set_compound(settings, "archive_files", "backend", "ARCHIVE_FILES_BACKEND") @@ -623,6 +630,9 @@ def configure(settings=None): # Allow the packaging app to register any services it has. config.include(".packaging") + # Configure TUF (RSTUF) + config.include(".tuf") + # Configure redirection support config.include(".redirects") diff --git a/warehouse/packaging/tasks.py b/warehouse/packaging/tasks.py index 3fe99edd4ead..95d9168fd6e1 100644 --- a/warehouse/packaging/tasks.py +++ b/warehouse/packaging/tasks.py @@ -27,6 +27,7 @@ from warehouse.metrics import IMetricsService from warehouse.packaging.interfaces import IFileStorage from warehouse.packaging.models import Description, File, Project, Release, Role +from warehouse.packaging.utils import render_simple_detail from warehouse.utils import readme logger = logging.getLogger(__name__) @@ -508,3 +509,9 @@ def populate_data_using_schema(file): json_rows, table_name, job_config=LoadJobConfig(schema=table_schema) ).result() break + + +@tasks.task(ignore_result=True, acks_late=True) +def generate_projects_simple_detail(request, projects): + for project in projects: + render_simple_detail(project, request, store=True) diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 5fb612b4833b..4e8226758ed0 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -93,6 +93,7 @@ def render_simple_detail(project, request, store=False): simple_detail_path = ( f"{project.normalized_name}/{content_hash}.{project.normalized_name}.html" ) + simple_detail_size = len(content.encode("utf-8")) if store: storage = request.find_service(ISimpleStorage) @@ -119,4 +120,4 @@ def render_simple_detail(project, request, store=False): }, ) - return (content_hash, simple_detail_path) + return (content_hash, simple_detail_path, simple_detail_size) diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py new file mode 100644 index 000000000000..45d74f3c5061 --- /dev/null +++ b/warehouse/tuf/__init__.py @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def includeme(config): + api_base_url = config.registry.settings["tuf.api.url"] + config.add_settings({"tuf.api.task.url": f"{api_base_url}task/"}) + config.add_settings({"tuf.api.publish.url": f"{api_base_url}targets/publish/"})