TUF Initialization using python-tuf 2.0.0

.gitignore

@@ -20,6 +20,7 @@ docker-compose.override.yaml
 
 node_modules/
 
+dev/tufkeys/
 dev/example.sql
 dev/prod.sql
 dev/prod.sql.xz
@@ -29,6 +30,7 @@ warehouse/.commit
 warehouse/static/components
 warehouse/static/dist
 warehouse/admin/static/dist
+warehouse/tuf/dist
 
 tags
 *.sw*

Makefile

@@ -1,5 +1,6 @@
 DB := example
 IPYTHON := no
+WAREHOUSE_CLI := docker-compose run --rm web python -m warehouse
 
 # set environment variable WAREHOUSE_IPYTHON_SHELL=1 if IPython
 # needed in development environment
@@ -94,6 +95,19 @@ initdb: .state/docker-build-web
 	docker-compose run --rm web python -m warehouse classifiers sync
 	$(MAKE) reindex
 
+inittuf:
+	$(WAREHOUSE_CLI) tuf dev keypair --name root --path /opt/warehouse/src/dev/tufkeys/root
+	$(WAREHOUSE_CLI) tuf dev keypair --name snapshot --path /opt/warehouse/src/dev/tufkeys/snapshot
+	$(WAREHOUSE_CLI) tuf dev keypair --name targets --path /opt/warehouse/src/dev/tufkeys/targets1
+	$(WAREHOUSE_CLI) tuf dev keypair --name targets --path /opt/warehouse/src/dev/tufkeys/targets2
+	$(WAREHOUSE_CLI) tuf dev keypair --name timestamp --path /opt/warehouse/src/dev/tufkeys/timestamp
+	$(WAREHOUSE_CLI) tuf dev keypair --name bins --path /opt/warehouse/src/dev/tufkeys/bins
+	$(WAREHOUSE_CLI) tuf dev keypair --name bin-n --path /opt/warehouse/src/dev/tufkeys/bin-n
+	$(WAREHOUSE_CLI) tuf dev init-repo
+	$(WAREHOUSE_CLI) tuf dev init-delegations
+	$(WAREHOUSE_CLI) tuf dev add-all-packages
+	$(WAREHOUSE_CLI) tuf dev add-all-indexes
+
 reindex: .state/docker-build-web
 	docker-compose run --rm web python -m warehouse search reindex
 
@@ -102,6 +116,7 @@ shell: .state/docker-build-web
 
 clean:
 	rm -rf dev/*.sql
+	rm -rf dev/tufkeys
 
 purge: stop clean
 	rm -rf .state

Procfile

@@ -4,3 +4,4 @@ web-uploads: bin/start-web ddtrace-run python -m gunicorn.app.wsgiapp -c gunicor
 worker: bin/start-worker celery -A warehouse worker -Q default -l info --max-tasks-per-child 32
 worker-malware: bin/start-worker celery -A warehouse worker -Q malware -l info --max-tasks-per-child 32
 worker-beat: bin/start-worker celery -A warehouse beat -S redbeat.RedBeatScheduler -l info
+worker-tuf: bin/start-worker celery -A warehouse worker -Q tuf -l info --max-tasks-per-child 32

dev/environment

@@ -57,3 +57,14 @@ TWOFACTORREQUIREMENT_ENABLED=true
 TWOFACTORMANDATE_AVAILABLE=true
 TWOFACTORMANDATE_ENABLED=true
 OIDC_ENABLED=true
+
+TUF_URL="http://{request.domain}:9001/metadata"
+TUF_KEY_BACKEND=warehouse.tuf.services.LocalKeyService key.path=/opt/warehouse/src/dev
+TUF_STORAGE_BACKEND=warehouse.tuf.services.LocalStorageService
+TUF_REPOSITORY_BACKEND=warehouse.tuf.services.RepositoryService repo.path=/var/opt/warehouse/tuf_metadata
+TUF_ROOT_SECRET="an insecure private key password"
+TUF_SNAPSHOT_SECRET="an insecure private key password"
+TUF_TARGETS_SECRET="an insecure private key password"
+TUF_TIMESTAMP_SECRET="an insecure private key password"
+TUF_BINS_SECRET="an insecure private key password"
+TUF_BIN_N_SECRET="an insecure private key password"

docker-compose.yml

@@ -4,6 +4,7 @@ volumes:
   simple:
   packages:
   sponsorlogos:
+  tuf_metadata:
   vault:
 
 services:
@@ -88,6 +89,7 @@ services:
       # The :z option fixes permission issues with SELinux by setting a
       # permissive security context.
       - ./dev:/opt/warehouse/src/dev:z
+      - ./dev/tufkeys:/opt/warehouse/src/dev/tufkeys:z
       - ./docs:/opt/warehouse/src/docs:z
       - ./warehouse:/opt/warehouse/src/warehouse:z
       - ./tests:/opt/warehouse/src/tests:z
@@ -96,6 +98,7 @@ services:
       - packages:/var/opt/warehouse/packages
       - sponsorlogos:/var/opt/warehouse/sponsorlogos
       - simple:/var/opt/warehouse/simple
+      - tuf_metadata:/var/opt/warehouse/tuf_metadata
       - ./bin:/opt/warehouse/src/bin:z
       - ./requirements:/opt/warehouse/src/requirements:z
     ports:
@@ -119,6 +122,7 @@ services:
       - packages:/var/opt/warehouse/packages
       - sponsorlogos:/var/opt/warehouse/sponsorlogos
       - simple:/var/opt/warehouse/simple
+      - tuf_metadata:/var/opt/warehouse/metadata
     ports:
       - "9001:9001"
 
@@ -129,7 +133,9 @@ services:
         DEVEL: "yes"
     command: hupper -m celery -A warehouse worker -B -S redbeat.RedBeatScheduler -l info
     volumes:
+      - ./dev:/opt/warehouse/src/dev:z
       - ./warehouse:/opt/warehouse/src/warehouse:z
+      - tuf_metadata:/var/opt/warehouse/tuf_metadata
     env_file: dev/environment
     environment:
       C_FORCE_ROOT: "1"

requirements/main.in

@@ -53,6 +53,7 @@ requests
 requests-aws4auth
 redis>=2.8.0,<5.0.0
 rfc3986
+securesystemslib
 sentry-sdk
 setuptools
 sqlalchemy[asyncio]>=0.9,<1.5.0  # https://github.com/pypi/warehouse/pull/9228
@@ -63,6 +64,7 @@ stripe
 structlog
 transaction
 trove-classifiers
+tuf==2.0.0
 typeguard
 webauthn>=1.0.0,<2.0.0
 whitenoise

requirements/main.txt

@@ -1238,6 +1238,7 @@ requests==2.28.1 \
     #   premailer
     #   requests-aws4auth
     #   stripe
+    #   tuf
 requests-aws4auth==1.1.2 \
     --hash=sha256:23b7a054326f80f86caf87e3eaf54ea41aa27adbed4297bd3456b1fa38f06a52 \
     --hash=sha256:ebde0662dccda5023546055ec4cbe4470cae017ecbfce8d368b80b5e4a94d619
@@ -1258,6 +1259,12 @@ sentry-sdk==1.9.7 \
     --hash=sha256:af0987fc074ada4a166bdc7e9d99d1da7811c6107e4b3416c7052ea1adb77dfc \
     --hash=sha256:d391204a2a59c54b764cd351c44c67eed17b43d51f4dbe3eba48b83b70c93db9
     # via -r requirements/main.in
+securesystemslib==0.22.0 \
+    --hash=sha256:2f58ca1ee30fde5401300fe3b3841adcf7b4369674247fa63b258e07e1f52fd2 \
+    --hash=sha256:c3fc41ac32fe8bc9744b89e6ce2ebca45f4417ca737beb766a41c6cb21935662
+    # via
+    #   -r requirements/main.in
+    #   tuf
 six==1.16.0 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
     --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
@@ -1360,6 +1367,10 @@ trove-classifiers==2022.8.31 \
     --hash=sha256:0db52e6a5cbe1035f306fcfee0066f22bcf842004f19dcc6258e309e36e5eb5f \
     --hash=sha256:9e32190e4ec0b7a173789ee0db20c433ef25060e98f64ff32ae4111990085526
     # via -r requirements/main.in
+tuf==2.0.0 \
+    --hash=sha256:1524b0fbd8504245f600f121daf86b8fdcb30df74410acc9655944c4868e461c \
+    --hash=sha256:76e7f2a7aced84466865fac2a7127b6085afae51d4328af896fb46f952dd3a53
+    # via -r requirements/main.in
 typeguard==2.13.3 \
     --hash=sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4 \
     --hash=sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1

tests/conftest.py

@@ -36,6 +36,7 @@
 from pytest_postgresql.config import get_config
 from pytest_postgresql.janitor import DatabaseJanitor
 from sqlalchemy import event
+from tuf.api.metadata import StorageBackendInterface
 
 import warehouse
 
@@ -51,6 +52,8 @@
 from warehouse.organizations.interfaces import IOrganizationService
 from warehouse.subscriptions import services as subscription_services
 from warehouse.subscriptions.interfaces import IBillingService, ISubscriptionService
+from warehouse.tuf.interfaces import IKeyService
+from warehouse.tuf.services import RepositoryService
 
 from .common.db import Session
 from .common.db.accounts import EmailFactory, UserFactory
@@ -255,10 +258,14 @@ def app_config(database):
         "sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage",
         "billing.backend": "warehouse.subscriptions.services.MockStripeBillingService",
         "mail.backend": "warehouse.email.services.SMTPEmailSender",
+        "tuf.storage_backend": "warehouse.tuf.services.LocalStorageService",
+        "tuf.key_backend": "warehouse.tuf.services.LocalKeyService",
+        "tuf.repository_backend": "warehouse.tuf.services.RepositoryService",
         "malware_check.backend": (
             "warehouse.malware.services.PrinterMalwareCheckService"
         ),
         "files.url": "http://localhost:7000/",
+        "tuf.url": "http://localhost:7000/metadata/",
         "sessions.secret": "123456",
         "sessions.url": "redis://localhost:0/",
         "statuspage.url": "https://2p66nmmycsj3.statuspage.io",
@@ -445,6 +452,36 @@ def xmlrpc(self, path, method, *args):
         return xmlrpc.client.loads(resp.body)
 
 
+@pytest.fixture
+def tuf_repository(db_request):
+    class FakeStorageBackend(StorageBackendInterface):
+        pass
+
+    class FakeKeyBackend(IKeyService):
+        pass
+
+    db_request.registry.settings = {
+        "tuf.keytype": "ed25519",
+        "tuf.root.threshold": 1,
+        "tuf.root.expiry": 31536000,
+        "tuf.snapshot.threshold": 1,
+        "tuf.snapshot.expiry": 86400,
+        "tuf.targets.threshold": 2,
+        "tuf.targets.expiry": 31536000,
+        "tuf.timestamp.threshold": 1,
+        "tuf.timestamp.expiry": 86400,
+        "tuf.bins.threshold": 1,
+        "tuf.bins.expiry": 31536000,
+        "tuf.bin-n.threshold": 1,
+        "tuf.bin-n.expiry": 604800,
+    }
+
+    tuf_repo = RepositoryService(
+        FakeStorageBackend, FakeKeyBackend, db_request.registry.settings
+    )
+    return tuf_repo
+
+
 @pytest.fixture
 def webtest(app_config):
     # TODO: Ensure that we have per test isolation of the database level