[WIP] warehouse: TUF initialization

.gitignore

@@ -19,6 +19,7 @@ docker-compose.override.yaml
 
 node_modules/
 
+dev/tuf.*
 dev/example.sql
 dev/prod.sql
 dev/prod.sql.xz
@@ -28,6 +29,7 @@ warehouse/.commit
 warehouse/static/components
 warehouse/static/dist
 warehouse/admin/static/dist
+warehouse/tuf/dist
 
 tags
 *.sw*

Makefile

@@ -5,6 +5,7 @@ BRANCH := $(shell echo "$${TRAVIS_BRANCH:-master}")
 DB := example
 IPYTHON := no
 LOCALES := $(shell .state/env/bin/python -c "from warehouse.i18n import KNOWN_LOCALES; print(' '.join(set(KNOWN_LOCALES)-{'en'}))")
+WAREHOUSE_CLI := docker-compose run --rm web python -m warehouse
 
 # set environment variable WAREHOUSE_IPYTHON_SHELL=1 if IPython
 # needed in development environment
@@ -153,6 +154,16 @@ initdb:
 	docker-compose run --rm web python -m warehouse db upgrade head
 	$(MAKE) reindex
 
+inittuf:
+	$(WAREHOUSE_CLI) tuf keypair --name root --path /opt/warehouse/src/dev/tuf.root
+	$(WAREHOUSE_CLI) tuf keypair --name snapshot --path /opt/warehouse/src/dev/tuf.snapshot
+	$(WAREHOUSE_CLI) tuf keypair --name targets --path /opt/warehouse/src/dev/tuf.targets
+	$(WAREHOUSE_CLI) tuf keypair --name timestamp --path /opt/warehouse/src/dev/tuf.timestamp
+	$(WAREHOUSE_CLI) tuf keypair --name bins --path /opt/warehouse/src/dev/tuf.bins
+	$(WAREHOUSE_CLI) tuf keypair --name bin-n --path /opt/warehouse/src/dev/tuf.bin-n
+	$(WAREHOUSE_CLI) tuf new-repo
+	$(WAREHOUSE_CLI) tuf build-targets
+
 reindex:
 	docker-compose run --rm web python -m warehouse search reindex
 

Procfile

@@ -4,3 +4,4 @@ web-uploads: bin/start-web python -m gunicorn.app.wsgiapp -c gunicorn-uploads.co
 worker: bin/start-worker celery -A warehouse worker -Q default -l info --max-tasks-per-child 32
 worker-malware: bin/start-worker celery -A warehouse worker -Q malware -l info --max-tasks-per-child 32
 worker-beat: bin/start-worker celery -A warehouse beat -S redbeat.RedBeatScheduler -l info
+worker-tuf: bin/start-worker celery -A warehouse worker -Q tuf -l info --max-tasks-per-child 32

dev/environment

@@ -43,3 +43,13 @@ WAREHOUSE_LEGACY_DOMAIN=pypi.python.org
 
 VAULT_URL="http://vault:8200"
 VAULT_TOKEN="an insecure vault access token"
+
+TUF_KEY_BACKEND=warehouse.tuf.services.LocalKeyService key.path=/opt/warehouse/src/dev
+TUF_STORAGE_BACKEND=warehouse.tuf.services.LocalStorageService
+TUF_REPO_BACKEND=warehouse.tuf.services.LocalRepositoryService repo.path=/opt/warehouse/src/warehouse/tuf/dist
+TUF_ROOT_SECRET="an insecure private key password"
+TUF_SNAPSHOT_SECRET="an insecure private key password"
+TUF_TARGETS_SECRET="an insecure private key password"
+TUF_TIMESTAMP_SECRET="an insecure private key password"
+TUF_BINS_SECRET="an insecure private key password"
+TUF_BIN_N_SECRET="an insecure private key password"

docker-compose.yml

@@ -105,6 +105,7 @@ services:
         DEVEL: "yes"
     command: hupper -m celery -A warehouse worker -B -S redbeat.RedBeatScheduler -l info
     volumes:
+      - ./dev:/opt/warehouse/src/dev:z
       - ./warehouse:/opt/warehouse/src/warehouse:z
     env_file: dev/environment
     environment:

requirements/main.in

@@ -54,6 +54,7 @@ stdlib-list
 structlog
 transaction
 trove-classifiers
+tuf==0.15.0
 typeguard
 webauthn
 whitenoise

requirements/main.txt

@@ -696,7 +696,7 @@ pyramid==1.10.5 \
 python-dateutil==2.8.1 \
     --hash=sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c \
     --hash=sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a \
-    # via alembic, botocore, celery-redbeat, elasticsearch-dsl
+    # via alembic, botocore, celery-redbeat, elasticsearch-dsl, securesystemslib
 python-editor==1.0.4 \
     --hash=sha256:1bf6e860a8ad52a14c3ee1252d5dc25b2030618ed80c022598f00176adc8367d \
     --hash=sha256:51fda6bcc5ddbbb7063b2af7509e43bd84bfc32a4ff71349ec7847713882327b \
@@ -740,14 +740,18 @@ s3transfer==0.3.3 \
     --hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \
     --hash=sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db \
     # via boto3
+securesystemslib==0.16.0 \
+    --hash=sha256:3c3b44140a6729ed014dc0591d803848fc4fc95652300db6467d45c5ff11ba5c \
+    --hash=sha256:72b72d2c86668d4cfdd8f5c73c84121cff7c93d9bc3eaddb652425c9c091f675 \
+    # via tuf
 sentry-sdk==0.19.4 \
     --hash=sha256:1052f0ed084e532f66cb3e4ba617960d820152aee8b93fc6c05bd53861768c1c \
     --hash=sha256:4c42910a55a6b1fe694d5e4790d5188d105d77b5a6346c1c64cbea8c06c0e8b7 \
     # via -r requirements/main.in
 six==1.15.0 \
     --hash=sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259 \
     --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced \
-    # via argon2-cffi, automat, bcrypt, bleach, cryptography, elasticsearch-dsl, google-api-core, google-auth, google-cloud-bigquery, google-resumable-media, grpcio, html5lib, limits, packaging, protobuf, pymacaroons, pynacl, pyopenssl, python-dateutil, readme-renderer, structlog, tenacity, webauthn
+    # via argon2-cffi, automat, bcrypt, bleach, cryptography, elasticsearch-dsl, google-api-core, google-auth, google-cloud-bigquery, google-resumable-media, grpcio, html5lib, limits, packaging, protobuf, pymacaroons, pynacl, pyopenssl, python-dateutil, readme-renderer, securesystemslib, structlog, tenacity, tuf, webauthn
 sqlalchemy-citext==1.7.0 \
     --hash=sha256:69ba00f5505f92a1455a94eefc6d3fcf72dda3691ab5398a0b4d0d8d85bd6aab \
     # via -r requirements/main.in
@@ -815,6 +819,10 @@ trove-classifiers==2020.10.21 \
     --hash=sha256:ad33e51c5aaf5fdd25bf0571f12cb67b885bf73db25a0ca629b66acbd58b654d \
     --hash=sha256:b8c4684ca08ec3ecbbd229e16ee36821f3dc3ffc4c511a47f231b445fab7c31b \
     # via -r requirements/main.in
+tuf==0.15.0 \
+    --hash=sha256:a3bb7a86cecf9d5356666ce14378d6f39151720547fb9cf2cc4e1497b340c567 \
+    --hash=sha256:e0653e1339031d018212d593879f96152af212aaf07a205ebcfc65d62f76679c \
+    # via -r requirements/main.in
 typeguard==2.10.0 \
     --hash=sha256:a75c6d86ac9d1faf85c5ae952de473e5d26824dda6d4394ff6bc676849cfb939 \
     --hash=sha256:d830132dcd544d3f8a2a842ea739eaa0d7c099fcebb9dcdf3802f4c9929d8191 \

tests/conftest.py

@@ -179,6 +179,8 @@ def app_config(database):
         "files.backend": "warehouse.packaging.services.LocalFileStorage",
         "docs.backend": "warehouse.packaging.services.LocalFileStorage",
         "mail.backend": "warehouse.email.services.SMTPEmailSender",
+        "tuf.key_backend": "warehouse.tuf.services.LocalKeyService",
+        "tuf.repo_backend": "warehouse.tuf.services.LocalRepositoryService",
         "malware_check.backend": (
             "warehouse.malware.services.PrinterMalwareCheckService"
         ),

tests/unit/test_config.py

@@ -324,6 +324,7 @@ def __init__(self):
             pretend.call(".malware"),
             pretend.call(".manage"),
             pretend.call(".packaging"),
+            pretend.call(".tuf"),
             pretend.call(".redirects"),
             pretend.call(".routes"),
             pretend.call(".admin"),
@@ -374,7 +375,8 @@ def __init__(self):
         ),
     ]
     assert configurator_obj.add_static_view.calls == [
-        pretend.call("static", "warehouse:static/dist/", cache_max_age=315360000)
+        pretend.call("tuf", "warehouse:tuf/dist/metadata.staged/"),
+        pretend.call("static", "warehouse:static/dist/", cache_max_age=315360000),
     ]
     assert configurator_obj.add_cache_buster.calls == [
         pretend.call("warehouse:static/dist/", cachebuster_obj)

tests/unit/test_tasks.py

@@ -504,8 +504,12 @@ def test_includeme(env, ssl, broker_url, expected_url, transport_options):
         "task_queues": (
             Queue("default", routing_key="task.#"),
             Queue("malware", routing_key="malware.#"),
+            Queue("tuf", routing_key="tuf.#"),
         ),
-        "task_routes": {"warehouse.malware.tasks.*": {"queue": "malware"}},
+        "task_routes": {
+            "warehouse.malware.tasks.*": {"queue": "malware"},
+            "warehouse.tuf.tasks.*": {"queue": "tuf"},
+        },
         "REDBEAT_REDIS_URL": (config.registry.settings["celery.scheduler_url"]),
     }.items():
         assert app.conf[key] == value

warehouse/cli/tuf.py

@@ -0,0 +1,176 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import datetime
+
+import click
+
+from tuf import repository_tool
+
+from warehouse.cli import warehouse
+from warehouse.config import Environment
+from warehouse.tuf import utils
+from warehouse.tuf.constants import BIN_N_COUNT, TOPLEVEL_ROLES, Role
+
+
+def _make_backsigned_fileinfo_from_file(file):
+    return utils.make_fileinfo(file, custom={"backsigned": True})
+
+
+def _key_service(config):
+    key_service_class = config.maybe_dotted(config.registry.settings["tuf.key_backend"])
+    return key_service_class.create_service(None, config)
+
+
+def _repository_service(config):
+    repo_service_class = config.maybe_dotted(
+        config.registry.settings["tuf.repo_backend"]
+    )
+    return repo_service_class.create_service(None, config)
+
+
+def _set_expiration_for_role(config, role_obj, role_name):
+    # If we're initializing TUF for development purposes, give
+    # every role a long expiration time so that developers don't have to
+    # continually re-initialize it.
+    if config.registry.settings["warehouse.env"] == Environment.development:
+        role_obj.expiration = datetime.datetime.now() + datetime.timedelta(
+            seconds=config.registry.settings["tuf.development_metadata_expiry"]
+        )
+    else:
+        role_obj.expiration = datetime.datetime.now() + datetime.timedelta(
+            seconds=config.registry.settings[f"tuf.{role_name}.expiry"]
+        )
+
+
+@warehouse.group()  # pragma: no-branch
+def tuf():
+    """
+    Manage Warehouse's TUF state.
+    """
+
+
+@tuf.command()
+@click.pass_obj
+@click.option("--name", "name_", help="The name of the TUF role for this keypair")
+@click.option("--path", "path_", help="The basename of the Ed25519 keypair to generate")
+def keypair(config, name_, path_):
+    """
+    Generate a new TUF keypair, for development purposes.
+    """
+
+    repository_tool.generate_and_write_ed25519_keypair(
+        path_, password=config.registry.settings[f"tuf.{name_}.secret"]
+    )
+
+
+@tuf.command()
+@click.pass_obj
+def new_repo(config):
+    """
+    Initialize the TUF repository from scratch, including a brand new root.
+    """
+
+    repository = repository_tool.create_new_repository(
+        config.registry.settings["tuf.repo.path"]
+    )
+
+    key_service = _key_service(config)
+    for role in TOPLEVEL_ROLES:
+        role_obj = getattr(repository, role)
+        role_obj.threshold = config.registry.settings[f"tuf.{role}.threshold"]
+        _set_expiration_for_role(config, role_obj, role)
+
+        pubkeys = key_service.pubkeys_for_role(role)
+        privkeys = key_service.privkeys_for_role(role)
+        if len(pubkeys) < role_obj.threshold or len(privkeys) < role_obj.threshold:
+            raise click.ClickException(
+                f"Unable to initialize TUF repo ({role} needs {role_obj.threshold} keys"
+            )
+
+        for pubkey in pubkeys:
+            role_obj.add_verification_key(pubkey)
+
+        for privkey in privkeys:
+            role_obj.load_signing_key(privkey)
+
+    repository.mark_dirty(TOPLEVEL_ROLES)
+    repository.writeall(
+        consistent_snapshot=True,
+    )
+
+
+@tuf.command()
+@click.pass_obj
+def build_targets(config):
+    """
+    Given an initialized (but empty) TUF repository, create the delegated
+    targets role (bins) and its hashed bin delegations (each bin-n).
+    """
+
+    repo_service = _repository_service(config)
+    repository = repo_service.load_repository()
+
+    # Load signing keys. We do this upfront for the top-level roles.
+    key_service = _key_service(config)
+    for role in ["snapshot", "targets", "timestamp"]:
+        role_obj = getattr(repository, role)
+
+        [role_obj.load_signing_key(k) for k in key_service.privkeys_for_role(role)]
+
+    # NOTE: TUF normally does delegations by path patterns (i.e., globs), but PyPI
+    # doesn't store its uploads on the same logical host as the TUF repository.
+    # The last parameter to `delegate` is a special sentinel for this.
+    repository.targets.delegate(
+        Role.BINS.value, key_service.pubkeys_for_role(Role.BINS.value), ["*"]
+    )
+    bins_role = repository.targets(Role.BINS.value)
+    _set_expiration_for_role(config, bins_role, Role.BINS.value)
+
+    for privkey in key_service.privkeys_for_role(Role.BINS.value):
+        bins_role.load_signing_key(privkey)
+
+    bins_role.delegate_hashed_bins(
+        [],
+        key_service.pubkeys_for_role(Role.BIN_N.value),
+        BIN_N_COUNT,
+    )
+
+    dirty_roles = ["snapshot", "targets", "timestamp", Role.BINS.value]
+    for bin_n_role in bins_role.delegations:
+        _set_expiration_for_role(config, bin_n_role, Role.BIN_N.value)
+        dirty_roles.append(bin_n_role.rolename)
+
+    for privkey in key_service.privkeys_for_role(Role.BIN_N.value):
+        for bin_n_role in bins_role.delegations:
+            bin_n_role.load_signing_key(privkey)
+
+    # Collect the "paths" for every PyPI package. These are packages already in
+    # existence, so we'll add some additional data to their targets to
+    # indicate that we're back-signing them.
+    from warehouse.db import Session
+    from warehouse.packaging.models import File
+
+    db = Session(bind=config.registry["sqlalchemy.engine"])
+    for file in db.query(File).all():
+        fileinfo = _make_backsigned_fileinfo_from_file(file)
+        bins_role.add_target_to_bin(
+            file.path,
+            number_of_bins=BIN_N_COUNT,
+            fileinfo=fileinfo,
+        )
+
+    repository.mark_dirty(dirty_roles)
+    repository.writeall(
+        consistent_snapshot=True,
+        use_existing_fileinfo=True,
+    )

warehouse/config.py

@@ -214,13 +214,22 @@ def configure(settings=None):
         coercer=int,
         default=21600,  # 6 hours
     )
+    maybe_set(settings, "tuf.root.secret", "TUF_ROOT_SECRET")
+    maybe_set(settings, "tuf.snapshot.secret", "TUF_SNAPSHOT_SECRET")
+    maybe_set(settings, "tuf.targets.secret", "TUF_TARGETS_SECRET")
+    maybe_set(settings, "tuf.timestamp.secret", "TUF_TIMESTAMP_SECRET")
+    maybe_set(settings, "tuf.bins.secret", "TUF_BINS_SECRET")
+    maybe_set(settings, "tuf.bin-n.secret", "TUF_BIN_N_SECRET")
     maybe_set_compound(settings, "files", "backend", "FILES_BACKEND")
     maybe_set_compound(settings, "docs", "backend", "DOCS_BACKEND")
     maybe_set_compound(settings, "origin_cache", "backend", "ORIGIN_CACHE")
     maybe_set_compound(settings, "mail", "backend", "MAIL_BACKEND")
     maybe_set_compound(settings, "metrics", "backend", "METRICS_BACKEND")
     maybe_set_compound(settings, "breached_passwords", "backend", "BREACHED_PASSWORDS")
     maybe_set_compound(settings, "malware_check", "backend", "MALWARE_CHECK_BACKEND")
+    maybe_set_compound(settings, "tuf", "key_backend", "TUF_KEY_BACKEND")
+    maybe_set_compound(settings, "tuf", "storage_backend", "TUF_STORAGE_BACKEND")
+    maybe_set_compound(settings, "tuf", "repo_backend", "TUF_REPO_BACKEND")
 
     # Add the settings we use when the environment is set to development.
     if settings["warehouse.env"] == Environment.development:
@@ -249,6 +258,10 @@ def configure(settings=None):
             ],
         )
 
+        # For development only: this artificially prolongs the expirations of any
+        # Warehouse-generated TUF metadata by approximately one year.
+        settings.setdefault("tuf.development_metadata_expiry", 31536000)
+
     # Actually setup our Pyramid Configurator with the values pulled in from
     # the environment as well as the ones passed in to the configure function.
     config = Configurator(settings=settings)
@@ -424,6 +437,13 @@ def configure(settings=None):
     # Allow the packaging app to register any services it has.
     config.include(".packaging")
 
+    # Register TUF support for package integrity
+    config.include(".tuf")
+
+    # Serve the TUF metadata files.
+    # TODO: This should be routed to the TUF GCS bucket.
+    config.add_static_view("tuf", "warehouse:tuf/dist/metadata.staged/")
+
     # Configure redirection support
     config.include(".redirects")
 

warehouse/forklift/legacy.py

@@ -62,6 +62,7 @@
     Role,
 )
 from warehouse.packaging.tasks import update_bigquery_release_files
+from warehouse.tuf.interfaces import IRepositoryService
 from warehouse.utils import http, readme
 
 ONE_MB = 1 * 1024 * 1024
@@ -1425,6 +1426,9 @@ def file_upload(request):
                 },
             )
 
+        repository = request.find_service(IRepositoryService)
+        repository.add_target(file_)
+
     # We are flushing the database requests so that we
     # can access the server default values when initiating celery
     # tasks.