PyPI/Warehouse using RSTUF

Adds the RSTUF in the Warehouse infrastructure * Include the RSTUF Ceremony payload file - It is generated using `rstuf admin ceremony`, and the keys * Add the development dependencies - RSTUF CLI and dependencies * Include RSTUF components to the `docker-compose.yml` - RSTUF uses the same Redis Server but uses unique Redis DB ids `1` and `2` - RSTUF uses the same PostgreSQL, but a specific database rstuf * Add the RSTUF environment configuration for development * Define the Makefile commands for RSTUF - `make tufinit` to bootstrap the RSTUF service - `make tufimport` to import all project packages to the RSTUF service * Define the basic commands for RSTUF within Warehouse - Command to import all existent packages and indexes to TUF metadata (`warehouse tuf dev import-all`) Signed-off-by: Kairo de Araujo <kdearaujo@vmware.com>
kairoaraujo · Jun 13, 2023 · 706478a · 706478a
1 parent d9c18a0
commit 706478a
Show file tree

Hide file tree

Showing 16 changed files with 426 additions and 2 deletions.
diff --git a/Makefile b/Makefile
@@ -117,6 +117,14 @@ reindex: .state/docker-build-base
 shell: .state/docker-build-base
 	docker compose run --rm web python -m warehouse shell
 
+tufinit:
+	docker compose run --rm web psql -h db -d postgres -U postgres -c "CREATE DATABASE rstuf ENCODING 'UTF8'"
+	docker compose restart rstuf-worker01 rstuf-worker02
+	docker compose run --rm web rstuf admin ceremony -b -u -f /opt/warehouse/src/dev/rstuf-bootstrap-payload.json --upload-server http://rstuf-api
+
+tufimport:
+	docker-compose run --rm web python -m warehouse tuf dev import-all
+
 dbshell: .state/docker-build-base
 	docker compose run --rm web psql -h db -d warehouse -U postgres
 
@@ -131,4 +139,4 @@ purge: stop clean
 stop:
 	docker compose stop
 
-.PHONY: default build serve initdb shell dbshell tests dev-docs user-docs deps clean purge debug stop compile-pot runmigrations
+.PHONY: default build serve initdb shell dbshell tests dev-docs user-docs deps clean purge debug stop compile-pot runmigrations tufinit tufimport
diff --git a/dev/environment b/dev/environment
@@ -68,3 +68,10 @@ OIDC_AUDIENCE=pypi
 # Default to the reCAPTCHA testing keys from https://developers.google.com/recaptcha/docs/faq
 RECAPTCHA_SITE_KEY=6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI
 RECAPTCHA_SECRET_KEY=6LeIxAcTAAAAAGG-vFI1TnRWxMZNFuojJ4WifJWe
+
+TUF_ENABLED=true
+TUF_METADATA_URL="http://files:9001/metadata/"
+TUF_API_URL="http://rstuf-api/api/v1/"
+TUF_DATABASE_URL="postgresql://postgres@db/rstuf"
+TUF_ROOT_SECRET="an insecure private key password"
+TUF_ONLINE_SECRET="an insecure private key password"
diff --git a/dev/rstuf-bootstrap-payload.json b/dev/rstuf-bootstrap-payload.json
@@ -0,0 +1,90 @@
+{
+  "settings": {
+    "expiration": {
+      "root": 365,
+      "targets": 365,
+      "snapshot": 1,
+      "timestamp": 1,
+      "bins": 1
+    },
+    "services": {
+      "number_of_delegated_bins": 256,
+      "targets_base_url": "\"http://127.0.0.1:9001/simple/\"/",
+      "targets_online_key": true
+    }
+  },
+  "metadata": {
+    "root": {
+      "signatures": [
+        {
+          "keyid": "a0cb8f1d00f8c7455e92272e01f551fc96c38d3b6bd201d7d3bdc08b3a418d1d",
+          "sig": "6fe3f661a40677df1ff5fac724cf3a47c826224be5ff9e1099cb76f826bac64722fa5e8120ad7eb032565a75a561d69255985b9de4ec25bb115710e8d3602d0b"
+        },
+        {
+          "keyid": "d5a3a5b1d77c59675fb830a558b7925a6b3e4da2e888af7372094984fbe37e9e",
+          "sig": "12485c76a748feed1ffdef59c24ba3258e56a20304207ae42138fff2c8c7314a14fb8f0beb7adfe85e78aebfc75200bac233a18a02d8c79ff06813f3900ff50e"
+        }
+      ],
+      "signed": {
+        "_type": "root",
+        "version": 1,
+        "spec_version": "1.0.30",
+        "expires": "2024-06-11T16:40:02Z",
+        "consistent_snapshot": true,
+        "keys": {
+          "a0cb8f1d00f8c7455e92272e01f551fc96c38d3b6bd201d7d3bdc08b3a418d1d": {
+            "keytype": "ed25519",
+            "scheme": "ed25519",
+            "keyval": {
+              "public": "ac5cd92ec491fea3f0b4c8a04af3fb957b5fc8965a79379131cfa4581905739f"
+            },
+            "name": "root key 1"
+          },
+          "d5a3a5b1d77c59675fb830a558b7925a6b3e4da2e888af7372094984fbe37e9e": {
+            "keytype": "ed25519",
+            "scheme": "ed25519",
+            "keyval": {
+              "public": "6d112f8658d1d8f42b17a263641bf7bd8940c97f25f9ea83d3aa609ec5fe9a91"
+            },
+            "name": "root key 2"
+          },
+          "64b5a379908148215a6bc1c9c66aa595fc87037555a054c4dddae5fc96d75bc2": {
+            "keytype": "ed25519",
+            "scheme": "ed25519",
+            "keyval": {
+              "public": "41df147e582fb6c14445da4db011b7d7d03824ea7b64aef5bb3aa8a57269b327"
+            },
+            "name": "online key v1"
+          }
+        },
+        "roles": {
+          "root": {
+            "keyids": [
+              "a0cb8f1d00f8c7455e92272e01f551fc96c38d3b6bd201d7d3bdc08b3a418d1d",
+              "d5a3a5b1d77c59675fb830a558b7925a6b3e4da2e888af7372094984fbe37e9e"
+            ],
+            "threshold": 1
+          },
+          "timestamp": {
+            "keyids": [
+              "64b5a379908148215a6bc1c9c66aa595fc87037555a054c4dddae5fc96d75bc2"
+            ],
+            "threshold": 1
+          },
+          "snapshot": {
+            "keyids": [
+              "64b5a379908148215a6bc1c9c66aa595fc87037555a054c4dddae5fc96d75bc2"
+            ],
+            "threshold": 1
+          },
+          "targets": {
+            "keyids": [
+              "64b5a379908148215a6bc1c9c66aa595fc87037555a054c4dddae5fc96d75bc2"
+            ],
+            "threshold": 1
+          }
+        }
+      }
+    }
+  }
+}
diff --git a/dev/tufkeys/online/online b/dev/tufkeys/online/online
@@ -0,0 +1 @@
+efd5e924987f59b3700a4188b83ae4dd@@@@100000@@@@c3cf5853b7cd2250cb72a1c0b4141c7367acddd3bcb6e96f6ec560f5f50e1c9e@@@@2480ed147201dd75b2d8f20b4a56534c@@@@58e914497e885078dbd8b5e6d10c1c435d8b1f83b53c35c8aa27fd5c703bcd3cc4b7ec8af0ef1d8444f5cad2a54093831944ac425777133f91c98df71018932f3fae77533f8f489f2bdbc63c0faddf2a00a63da37bd292f3ce7b35e86b7ddd90d0f2d92eaa9a264fda9eeb85f714b6745a9ff5ea3e3cd466d94557b0dce8fde3503a12ffd4ddfa0beaf5e509ce3514d071dc26af385dcd23f239711efbc86b2f736027f7940f9a3a786cb3a329158e5a0487ce50ee3a5ed4a032d6e556181a9ffb26c20800d0b4cd0b2149afee333986c0722101603f1d144d4df14a493376d516e7cbd54aa070f96c630672e8b2e8c89c7fe6d44d6cae87188289bdc8361635c613a06cd81f4b6f630938f003aa224c1ff3b31ec01c78fe5e85fcefcfe2d3beb33213b5e244c21b73841d46f47b9360e7bd6c75b8af6aa398dffdae5bce50795d57d37a3df0b3088392c1d050a46302
diff --git a/dev/tufkeys/online/online.pub b/dev/tufkeys/online/online.pub
@@ -0,0 +1 @@
+{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "41df147e582fb6c14445da4db011b7d7d03824ea7b64aef5bb3aa8a57269b327"}}
diff --git a/dev/tufkeys/root/root1 b/dev/tufkeys/root/root1
@@ -0,0 +1 @@
+0b81cc88ce39650626eba6a9b0420dad@@@@100000@@@@875fed5a914d59843f5280260145405c7599026e915efca75c7773b49b33b2e6@@@@bfb692965db58b72b5f1e9bceaeccc37@@@@81d62e800c1a22b8a82b7c795a2b77990494577dc098755b78d02eac175d3b96c5a5d4cc6e08c17410069118840c32443b7be43c3663963bfdf58f813cdcfd33a29770392ad35c05df7edbd51aaa0e6a6122b752cc876628633d10078fcf494818cf62abe41591ce9543abc4d0ea7a8e731097f1b97e2915358c015d61dadbdefe9a9be8138bdd87d89b952e89817951b60fee1089dcdadd4937e638571f47bd1258276803163797174dee1e963c731f40affaf35804117a5ee555767ea05bb165a3d4751a40d6e0dfc519049e2346dfcd09f851637328883251027ef3bb1566a1c4b1b95f127a636d0499a28fa52eade9e53d9560879ed56346582045750403978f575a4d753abd6e00e46c264f073e858693210ae04df8ce0520ca6fac64a668f61000e9a4b16c29cbcaf96dd35d5489bf1e36cbef3ee2927dffaf3ba822532653542a4516c3cacb3db4e2bca825cf
diff --git a/dev/tufkeys/root/root1.pub b/dev/tufkeys/root/root1.pub
@@ -0,0 +1 @@
+{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "ac5cd92ec491fea3f0b4c8a04af3fb957b5fc8965a79379131cfa4581905739f"}}
diff --git a/dev/tufkeys/root/root2 b/dev/tufkeys/root/root2
@@ -0,0 +1 @@
+370fd8314f62bf0743f150e9d9ec1883@@@@100000@@@@b89bb6032fa26cba87e51649a6e83df8a6ff92b81e2616a2af2f3794ce96aee2@@@@67c0d2cde784d5d354817ef42321d75d@@@@2558be0a0ed1b3db89be41669bb6657dcb85e05b1e41a2d57319fb715779cc230940ea88614cf96bf0f0f07a1f8726a780bf2003013c28c36956f597238d64502c3d063e8cf0d953f883f41f7787bcc7c233ca9e6c08e0fc0fdb988f99de80e456dc80d86f087a7535d5e6bed7db11feb4af247c04e01c3b7c0e658cfd6fb170e6370240cc7b0f9cfe0d15723122ae70c56d10487cc19b4dacb047ac8194cc1435a2e687bdcf20f2b4aca227581b3939b0c8aa712d8237bc8dbd977d64c4548359c75d0dad452e0a5517ce02da0db1bf5a077782e7997126f789e3ac93e3cf57feb08b9bf988ba8ff8b42dbf09bc5bbafbd8366d3ab1ebc0fe03899a48abdc55f324d96ef3a70265a728d5ce06f9d35c7b93ad399902be82f87f82f1eecbb1d4666031d515a2f9f14d6160560a9b505a444af8a79358e46609cfa339df8adacca9ee7cf7643b71da535032ae79934e7d
diff --git a/dev/tufkeys/root/root2.pub b/dev/tufkeys/root/root2.pub
@@ -0,0 +1 @@
+{"keytype": "ed25519", "scheme": "ed25519", "keyid_hash_algorithms": ["sha256", "sha512"], "keyval": {"public": "6d112f8658d1d8f42b17a263641bf7bd8940c97f25f9ea83d3aa609ec5fe9a91"}}
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -8,6 +8,8 @@ volumes:
   policies:
   vault:
   caches:
+  rstuf-worker-data:
+  rstuf-metadata:
 
 services:
   vault:
@@ -109,6 +111,8 @@ services:
       # Included to support linters during development
       - ./gunicorn-prod.conf.py:/opt/warehouse/src/gunicorn-prod.conf.py:z
       - ./gunicorn-uploads.conf.py:/opt/warehouse/src/gunicorn-uploads.conf.py:z
+      - rstuf-metadata:/var/opt/warehouse/metadata
+      - ./dev/tufkeys:/opt/warehouse/src/dev/tufkeys:z
 
   web:
     image: warehouse:docker-compose
@@ -138,6 +142,7 @@ services:
       - packages-archive:/var/opt/warehouse/packages-archive
       - sponsorlogos:/var/opt/warehouse/sponsorlogos
       - simple:/var/opt/warehouse/simple
+      - rstuf-metadata:/var/opt/warehouse/metadata
     ports:
       - "9001:9001"
 
@@ -156,6 +161,68 @@ services:
       ARCHIVE_FILES_BACKEND: "warehouse.packaging.services.LocalArchiveFileStorage path=/var/opt/warehouse/packages-archive/ url=http://files:9001/packages-archive/{path}"
       SIMPLE_BACKEND: "warehouse.packaging.services.LocalSimpleStorage path=/var/opt/warehouse/simple/ url=http://files:9001/simple/{path}"
 
+  rstuf-worker01:
+    image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:latest
+    volumes:
+      - rstuf-worker-data:/data
+      - ./dev/rstuf-workers-supervisor.conf:/opt/repository-service-tuf/supervisor.conf:z
+      - rstuf-metadata:/var/opt/repository-service-tuf/storage
+      - ./dev/tufkeys/online:/var/opt/repository-service-tuf/keystorage
+    environment:
+      - RSTUF_STORAGE_BACKEND=LocalStorage
+      - RSTUF_LOCAL_STORAGE_BACKEND_PATH=/var/opt/repository-service-tuf/storage
+      - RSTUF_KEYVAULT_BACKEND=LocalKeyVault
+      - RSTUF_LOCAL_KEYVAULT_PATH=/var/opt/repository-service-tuf/keystorage
+      - RSTUF_LOCAL_KEYVAULT_KEYS=online,an insecure private key password
+      - RSTUF_BROKER_SERVER=redis://redis/1
+      - RSTUF_REDIS_SERVER=redis://redis
+      - RSTUF_REDIS_SERVER_DB_RESULT=1
+      - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2
+      - RSTUF_SQL_SERVER=postgresql://postgres@db:5432/rstuf
+    healthcheck:
+      test: "exit 0"
+    restart: always
+    tty: true
+    depends_on:
+      db:
+        condition: service_healthy
+
+  rstuf-worker02:
+    image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:latest
+    volumes:
+      - rstuf-worker-data:/data
+      - ./dev/rstuf-workers-supervisor.conf:/opt/repository-service-tuf/supervisor.conf:z
+      - rstuf-metadata:/var/opt/repository-service-tuf/storage
+      - ./dev/tufkeys/online:/var/opt/repository-service-tuf/keystorage
+    environment:
+      - RSTUF_STORAGE_BACKEND=LocalStorage
+      - RSTUF_LOCAL_STORAGE_BACKEND_PATH=/var/opt/repository-service-tuf/storage
+      - RSTUF_KEYVAULT_BACKEND=LocalKeyVault
+      - RSTUF_LOCAL_KEYVAULT_PATH=/var/opt/repository-service-tuf/keystorage
+      - RSTUF_LOCAL_KEYVAULT_KEYS=online,an insecure private key password
+      - RSTUF_BROKER_SERVER=redis://redis/1
+      - RSTUF_REDIS_SERVER=redis://redis
+      - RSTUF_REDIS_SERVER_DB_RESULT=1
+      - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2
+      - RSTUF_SQL_SERVER=postgresql://postgres@db:5432/rstuf
+    healthcheck:
+      test: "exit 0"
+    restart: always
+    tty: true
+    depends_on:
+      db:
+        condition: service_healthy
+
+  rstuf-api:
+    image: ghcr.io/repository-service-tuf/repository-service-tuf-api:latest
+    ports:
+      - 8001:80
+    environment:
+      - RSTUF_BROKER_SERVER=redis://redis/1
+      - RSTUF_REDIS_SERVER=redis://redis
+      - RSTUF_REDIS_SERVER_DB_RESULT=1
+      - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2
+
   static:
     build:
       context: .

diff --git a/requirements/dev.txt b/requirements/dev.txt
@@ -2,3 +2,8 @@ asyncudp>=0.7
 hupper>=1.9
 pip-tools>=1.0
 pyramid_debugtoolbar>=2.5
+repository-service-tuf==0.3.0a1
+securesystemslib
+dynaconf
+rich-click
+commonmark