diff --git a/requirements/main.in b/requirements/main.in index b0682cc75112..2f21ec346fe9 100644 --- a/requirements/main.in +++ b/requirements/main.in @@ -62,7 +62,7 @@ stdlib-list structlog transaction trove-classifiers -tuf==1.1.0 +tuf==2.0.0 typeguard webauthn>=1.0.0,<2.0.0 whitenoise diff --git a/requirements/main.txt b/requirements/main.txt index eef73875e81e..24941b89f9ed 100644 --- a/requirements/main.txt +++ b/requirements/main.txt @@ -1270,9 +1270,9 @@ trove-classifiers==2022.8.7 \ --hash=sha256:10053f2df40092c2d65d750cf12fab6df28714e422114fed91465814939a3284 \ --hash=sha256:c0618efe27904e272de1212143c8554657eef8adcf9800f1d2a1cf7164cc185b # via -r requirements/main.in -tuf==1.1.0 \ - --hash=sha256:28cd35eafa5aa4223eba03a397d14acb57c522381180db9ff3b54477dcbe1b73 \ - --hash=sha256:512a864789e291b5e8f5a5ace0e87b2d158303364a77ad6e53ffd042ba2b4933 +tuf==2.0.0 \ + --hash=sha256:1524b0fbd8504245f600f121daf86b8fdcb30df74410acc9655944c4868e461c \ + --hash=sha256:76e7f2a7aced84466865fac2a7127b6085afae51d4328af896fb46f952dd3a53 # via -r requirements/main.in typeguard==2.13.3 \ --hash=sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4 \ diff --git a/tests/conftest.py b/tests/conftest.py index 39d3da7c661d..1e9f7001e1a8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -50,7 +50,7 @@ from warehouse.organizations import services as organization_services from warehouse.organizations.interfaces import IOrganizationService from warehouse.tuf.interfaces import IKeyService -from warehouse.tuf.repository import MetadataRepository +from warehouse.tuf.services import RepositoryService from .common.db import Session from .common.db.accounts import EmailFactory, UserFactory @@ -446,7 +446,7 @@ class FakeKeyBackend(IKeyService): "tuf.bin-n.expiry": 604800, } - tuf_repo = MetadataRepository( + tuf_repo = RepositoryService( FakeStorageBackend, FakeKeyBackend, db_request.registry.settings ) return tuf_repo diff --git a/tests/unit/cli/test_tuf.py b/tests/unit/cli/test_tuf.py index d70f767f7940..93b4c5fb561e 100644 --- a/tests/unit/cli/test_tuf.py +++ b/tests/unit/cli/test_tuf.py @@ -297,10 +297,9 @@ def test_add_all_indexes_content_hash_none(self, cli, monkeypatch): result = cli.invoke(add_all_indexes, obj=config) - assert result.exit_code == 0 + assert result.exit_code == 1 assert config.task.calls == [ pretend.call(_add_hashed_targets), - pretend.call(_add_hashed_targets), ] assert task.get_request.calls == [pretend.call()] diff --git a/tests/unit/tuf/test_hash_bins.py b/tests/unit/tuf/test_hash_bins.py deleted file mode 100644 index 70b776c58e72..000000000000 --- a/tests/unit/tuf/test_hash_bins.py +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from warehouse.tuf.hash_bins import HashBins - - -class TestHashBins: - def test_basic_init(self): - test_hash_bins = HashBins(32) - assert test_hash_bins.number_of_bins == 32 - assert test_hash_bins.prefix_len == 2 - assert test_hash_bins.number_of_prefixes == 256 - assert test_hash_bins.bin_size == 8 - - test_hash_bins = HashBins(16) - assert test_hash_bins.number_of_bins == 16 - assert test_hash_bins.prefix_len == 1 - assert test_hash_bins.number_of_prefixes == 16 - assert test_hash_bins.bin_size == 1 - - def test__bin_name(self): - test_hash_bins = HashBins(32) - assert test_hash_bins._bin_name(1, 7) == "01-07" - assert test_hash_bins._bin_name(32, 39) == "20-27" - - def test_generate(self): - test_hash_bins = HashBins(16) - hash_bin_list = [ - ("0", ["0"]), - ("1", ["1"]), - ("2", ["2"]), - ("3", ["3"]), - ("4", ["4"]), - ("5", ["5"]), - ("6", ["6"]), - ("7", ["7"]), - ("8", ["8"]), - ("9", ["9"]), - ("a", ["a"]), - ("b", ["b"]), - ("c", ["c"]), - ("d", ["d"]), - ("e", ["e"]), - ("f", ["f"]), - ] - for i in test_hash_bins.generate(): - assert i in hash_bin_list - - def test_get_delegate(self): - test_hash_bins = HashBins(128) - assert test_hash_bins.get_delegate("filepath0") == "24-25" - assert test_hash_bins.get_delegate("filepath1") == "d8-d9" diff --git a/tests/unit/tuf/test_repository.py b/tests/unit/tuf/test_repository.py deleted file mode 100644 index 2dae6561e207..000000000000 --- a/tests/unit/tuf/test_repository.py +++ /dev/null @@ -1,809 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import datetime - -import pretend -import pytest - -from securesystemslib.exceptions import StorageError -from tuf.api.metadata import ( - TOP_LEVEL_ROLE_NAMES, - MetaFile, - Snapshot, - StorageBackendInterface, -) - -from warehouse.tuf import repository -from warehouse.tuf.interfaces import IKeyService - - -class TestMetadataRepository: - def test_basic_init(self, db_request): - class FakeStorageBackend(StorageBackendInterface): - pass - - class FakeKeyBackend(IKeyService): - pass - - tuf_repository = repository.MetadataRepository( - FakeStorageBackend, FakeKeyBackend, db_request.registry.settings - ) - assert tuf_repository.storage_backend == FakeStorageBackend - assert tuf_repository.key_backend == FakeKeyBackend - - def test_is_initialized(self, tuf_repository): - - tuf_repository.load_role = pretend.call_recorder(lambda role: None) - - assert tuf_repository.is_initialized is False - - for role in TOP_LEVEL_ROLE_NAMES: - assert pretend.call(role) in tuf_repository.load_role.calls - - def test_is_initialized_load_metadata(self, tuf_repository): - tuf_repository.load_role = pretend.call_recorder(lambda role: True) - assert tuf_repository.is_initialized is True - assert tuf_repository.load_role.calls[0] in [ - pretend.call("targets"), - pretend.call("root"), - pretend.call("snapshot"), - pretend.call("timestamp"), - ] - - def test_is_initialized_cannot_load_metadata(self, tuf_repository): - tuf_repository.load_role = pretend.call_recorder(lambda role: None) - assert tuf_repository.is_initialized is False - assert tuf_repository.load_role.calls[0] in [ - pretend.call("targets"), - pretend.call("root"), - pretend.call("snapshot"), - pretend.call("timestamp"), - ] - - def test_is_initialized_raise_storageerror(self, tuf_repository): - tuf_repository.load_role = pretend.call_recorder(pretend.raiser(StorageError)) - assert tuf_repository.is_initialized is False - assert tuf_repository.load_role.calls[0] in [ - pretend.call("targets"), - pretend.call("root"), - pretend.call("snapshot"), - pretend.call("timestamp"), - ] - - def test__set_expiration_for_role(self, tuf_repository, monkeypatch): - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_datetime = pretend.stub(now=pretend.call_recorder(lambda: fake_time)) - monkeypatch.setattr("warehouse.tuf.repository.datetime", fake_datetime) - - result = tuf_repository._set_expiration_for_role(Snapshot.type) - assert str(result) == "2019-06-17 09:05:01" - assert fake_datetime.now.calls == [pretend.call()] - - def test__create_delegated_targets_roles(self, tuf_repository, monkeypatch): - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_targets_md = pretend.stub( - signed=pretend.stub( - delegations=None, add_key=pretend.call_recorder(lambda *a, **kw: None) - ) - ) - fake_snapshot_md = pretend.stub(signed=pretend.stub(meta={})) - - tuf_repository.load_role = pretend.call_recorder( - lambda role: fake_snapshot_md if role == Snapshot.type else None - ) - tuf_repository._store = pretend.call_recorder(lambda *a, **kw: None) - fake_signers = [ - pretend.stub( - key_dict={"keyid": "key1"}, - sign=pretend.call_recorder(lambda *a: "key1"), - ) - ] - - test_delegate_roles_parameters = [ - ( - repository.DelegatedRole( - "test_bin", - ["key1", "key2"], - 1, - False, - paths=["*/*"], - ), - fake_signers, - fake_time, - ) - ] - - fake_metadata = pretend.stub( - sign=pretend.call_recorder(lambda *a, **kw: None), - signed=pretend.stub(version=3), - ) - monkeypatch.setattr( - "warehouse.tuf.repository.Metadata", lambda *a, **kw: fake_metadata - ) - monkeypatch.setattr( - "warehouse.tuf.repository.Key.from_securesystemslib_key", - lambda *a, **kw: "fake_Key", - ) - result = tuf_repository._create_delegated_targets_roles( - delegator_metadata=fake_targets_md, - delegatees=test_delegate_roles_parameters, - ) - - assert "test_bin.json" in result.signed.meta - assert tuf_repository.load_role.calls == [ - pretend.call(Snapshot.type), - ] - assert tuf_repository._store.calls[0].args[0] == "test_bin" - - def test__create_delegated_targets_roles_with_snapshot_md( - self, tuf_repository, monkeypatch - ): - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_targets_md = pretend.stub( - signed=pretend.stub( - delegations=None, add_key=pretend.call_recorder(lambda *a, **kw: None) - ) - ) - fake_snapshot_md = pretend.stub(signed=pretend.stub(meta={})) - fake_signers = [ - pretend.stub( - key_dict={"keyid": "key1"}, - sign=pretend.call_recorder(lambda *a: "key1"), - ) - ] - - tuf_repository.load_role = pretend.call_recorder( - lambda role: fake_snapshot_md if role == Snapshot.type else None - ) - tuf_repository._store = pretend.call_recorder(lambda *a, **kw: None) - - test_delegate_roles_parameters = [ - ( - repository.DelegatedRole( - "test_bin", - ["key1", "key2"], - 1, - False, - paths=["*/*"], - ), - fake_signers, - fake_time, - ) - ] - - fake_metadata = pretend.stub( - sign=pretend.call_recorder(lambda *a, **kw: None), - signed=pretend.stub(version=3), - ) - monkeypatch.setattr( - "warehouse.tuf.repository.Metadata", lambda *a, **kw: fake_metadata - ) - monkeypatch.setattr( - "warehouse.tuf.repository.Key.from_securesystemslib_key", - lambda *a, **kw: None, - ) - result = tuf_repository._create_delegated_targets_roles( - delegator_metadata=fake_targets_md, - delegatees=test_delegate_roles_parameters, - snapshot_metadata=fake_snapshot_md, - ) - - assert "test_bin.json" in result.signed.meta - assert tuf_repository.load_role.calls == [] - assert tuf_repository._store.calls[0].args[0] == "test_bin" - - def test__create_delegated_targets_roles_has_delegations( - self, tuf_repository, monkeypatch - ): - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_targets_md = pretend.stub( - signed=pretend.stub( - delegations=pretend.stub(roles={"role1": "delegated_stuff"}), - add_key=pretend.call_recorder(lambda *a, **kw: None), - ) - ) - fake_snapshot_md = pretend.stub(signed=pretend.stub(meta={})) - fake_signers = [ - pretend.stub( - key_dict={"keyid": "key1"}, - sign=pretend.call_recorder(lambda *a: "key1"), - ) - ] - - tuf_repository.load_role = pretend.call_recorder( - lambda role: fake_snapshot_md if role == Snapshot.type else None - ) - tuf_repository._store = pretend.call_recorder(lambda *a, **kw: None) - - test_delegate_roles_parameters = [ - ( - repository.DelegatedRole( - "test_bin", - ["key1", "key2"], - 1, - False, - paths=["*/*"], - ), - fake_signers, - fake_time, - ) - ] - - fake_metadata = pretend.stub( - sign=pretend.call_recorder(lambda *a, **kw: None), - signed=pretend.stub(version=3), - ) - monkeypatch.setattr( - "warehouse.tuf.repository.Metadata", lambda *a, **kw: fake_metadata - ) - monkeypatch.setattr( - "warehouse.tuf.repository.Key.from_securesystemslib_key", - lambda *a, **kw: None, - ) - result = tuf_repository._create_delegated_targets_roles( - delegator_metadata=fake_targets_md, - delegatees=test_delegate_roles_parameters, - ) - - assert "test_bin.json" in result.signed.meta - assert tuf_repository.load_role.calls == [ - pretend.call(Snapshot.type), - ] - assert "role1" in fake_targets_md.signed.delegations.roles.keys() - assert "test_bin" in fake_targets_md.signed.delegations.roles.keys() - assert tuf_repository._store.calls[0].args[0] == "test_bin" - - def test__filename(self, tuf_repository): - - assert tuf_repository._filename("root", 1) == "1.root.json" - assert tuf_repository._filename("root", 2) == "2.root.json" - assert tuf_repository._filename("snapshot", 2) == "2.snapshot.json" - assert tuf_repository._filename("timestamp", 2) == "timestamp.json" - assert tuf_repository._filename("timestamp", 3) == "timestamp.json" - - def test__store(self, tuf_repository): - fake_metadata = pretend.stub( - to_file=pretend.call_recorder(lambda *a, **kw: None), - signed=pretend.stub(version=1), - ) - - result = tuf_repository._store("root", fake_metadata) - - assert result is None - assert fake_metadata.to_file.calls[0].args[0] == "1.root.json" - - def test_initialization(self, tuf_repository): - fake_key = { - "keytype": "ed25519", - "scheme": "ed25519", - "keyid": ( - "6dcd53f0a90fca17700f819e939a74b133aa5cd8619f3dc03228c0c68dcc2abb" - ), - "keyid_hash_algorithms": ["sha256", "sha512"], - "keyval": { - "public": ( - "c864d93b521d5851275a7b7c79fb0ac76311c206262eabd67319eba6665b1417" - ), - "private": ( - "bbe40143bfe1a3b6a41647f590e398fb8dd38fddf6b279edefdc022cdb649cdc" - ), - }, - } - fake_signers = [ - pretend.stub( - key_dict=fake_key, - sign=pretend.call_recorder(lambda *a: pretend.stub(keyid="key1")), - ), - pretend.stub( - key_dict=fake_key, - sign=pretend.call_recorder(lambda *a: pretend.stub(keyid="key2")), - ), - ] - - top_roles_payload = dict() - for role in TOP_LEVEL_ROLE_NAMES: - top_roles_payload[role] = fake_signers - - tuf_repository.load_role = pretend.call_recorder(lambda *a, **kw: None) - tuf_repository._store = pretend.call_recorder(lambda *a, **kw: None) - result = tuf_repository.initialize(top_roles_payload, store=True) - - assert sorted(list(result.keys())) == sorted(list(TOP_LEVEL_ROLE_NAMES)) - - for call_store in tuf_repository._store.calls: - assert call_store.args[0] in TOP_LEVEL_ROLE_NAMES - - for role in TOP_LEVEL_ROLE_NAMES: - assert pretend.call(role) in tuf_repository.load_role.calls - - def test_initialization_store_false(self, tuf_repository): - fake_key = { - "keytype": "ed25519", - "scheme": "ed25519", - "keyid": ( - "6dcd53f0a90fca17700f819e939a74b133aa5cd8619f3dc03228c0c68dcc2abb" - ), - "keyid_hash_algorithms": ["sha256", "sha512"], - "keyval": { - "public": ( - "c864d93b521d5851275a7b7c79fb0ac76311c206262eabd67319eba6665b1417" - ), - "private": ( - "bbe40143bfe1a3b6a41647f590e398fb8dd38fddf6b279edefdc022cdb649cdc" - ), - }, - } - fake_signers = [ - pretend.stub( - key_dict=fake_key, - sign=pretend.call_recorder(lambda *a: pretend.stub(keyid="key1")), - ), - pretend.stub( - key_dict=fake_key, - sign=pretend.call_recorder(lambda *a: pretend.stub(keyid="key2")), - ), - ] - - top_roles_payload = dict() - for role in TOP_LEVEL_ROLE_NAMES: - top_roles_payload[role] = fake_signers - - tuf_repository.load_role = pretend.call_recorder(lambda *a, **kw: None) - tuf_repository._store = pretend.call_recorder(lambda *a, **kw: None) - result = tuf_repository.initialize(top_roles_payload, store=False) - - assert sorted(list(result.keys())) == sorted(list(TOP_LEVEL_ROLE_NAMES)) - - for call_store in tuf_repository._store.calls: - assert call_store.args[0] in TOP_LEVEL_ROLE_NAMES - - for role in TOP_LEVEL_ROLE_NAMES: - assert pretend.call(role) in tuf_repository.load_role.calls - - def test_initialization_already_initialized(self, tuf_repository): - top_roles_payload = dict() - for role in TOP_LEVEL_ROLE_NAMES: - top_roles_payload[role] = [{"key1": "key1_data"}] - - tuf_repository.load_role = pretend.call_recorder(lambda *a, **kw: True) - with pytest.raises(FileExistsError) as err: - tuf_repository.initialize(top_roles_payload, store=False) - - assert "Metadata already exists in the Storage Service" in str(err.value) - assert tuf_repository.load_role.calls in [ - [pretend.call("targets")], - [pretend.call("root")], - [pretend.call("snapshot")], - [pretend.call("timestamp")], - ] - - def test_initialization_threshold_more_than_keys(self, tuf_repository): - fake_key = { - "keytype": "ed25519", - "scheme": "ed25519", - "keyid": ( - "6dcd53f0a90fca17700f819e939a74b133aa5cd8619f3dc03228c0c68dcc2abb" - ), - "keyid_hash_algorithms": ["sha256", "sha512"], - "keyval": { - "public": ( - "c864d93b521d5851275a7b7c79fb0ac76311c206262eabd67319eba6665b1417" - ), - "private": ( - "bbe40143bfe1a3b6a41647f590e398fb8dd38fddf6b279edefdc022cdb649cdc" - ), - }, - } - fake_signers = [ - pretend.stub( - key_dict=fake_key, - sign=pretend.call_recorder(lambda *a: pretend.stub(keyid="key1")), - ) - ] - top_roles_payload = dict() - for role in TOP_LEVEL_ROLE_NAMES: - top_roles_payload[role] = fake_signers - - tuf_repository.load_role = pretend.call_recorder(lambda *a, **kw: None) - tuf_repository._store = pretend.call_recorder(lambda *a, **kw: None) - - with pytest.raises(ValueError) as err: - tuf_repository.initialize(top_roles_payload, store=True) - - assert ("has missing Key(s) to match to defined threshold 2.") in str(err.value) - - for role in TOP_LEVEL_ROLE_NAMES: - assert pretend.call(role) in tuf_repository.load_role.calls - - def test_load_role(self, tuf_repository, monkeypatch): - fake_metadata = pretend.stub( - from_file=pretend.call_recorder(lambda *a, **kw: None), - ) - monkeypatch.setattr("warehouse.tuf.repository.Metadata", fake_metadata) - - tuf_repository.load_role("test_role_name") - assert fake_metadata.from_file.calls == [ - pretend.call("test_role_name", None, tuf_repository.storage_backend) - ] - - def test_delegate_targets_roles(self, tuf_repository): - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_key = { - "keytype": "ed25519", - "scheme": "ed25519", - "keyid": ( - "6dcd53f0a90fca17700f819e939a74b133aa5cd8619f3dc03228c0c68dcc2abb" - ), - "keyid_hash_algorithms": ["sha256", "sha512"], - "keyval": { - "public": ( - "c864d93b521d5851275a7b7c79fb0ac76311c206262eabd67319eba6665b1417" - ), - "private": ( - "bbe40143bfe1a3b6a41647f590e398fb8dd38fddf6b279edefdc022cdb649cdc" - ), - }, - } - fake_signers = [ - pretend.stub( - key_dict=fake_key, sign=pretend.call_recorder(lambda *a: "key1") - ) - ] - payload = {"xxxx-yyyy": fake_signers} - fake_targets_md = pretend.stub( - signed=pretend.stub( - delegations=None, - add_key=pretend.call_recorder(lambda *a, **kw: None), - expires=fake_time, - version=2, - ) - ) - fake_snapshot_md = pretend.stub(signed=pretend.stub(meta={})) - - tuf_repository.load_role = pretend.call_recorder( - lambda role: fake_snapshot_md if role == Snapshot.type else fake_targets_md - ) - tuf_repository._create_delegated_targets_roles = pretend.call_recorder( - lambda *a, **kw: fake_snapshot_md - ) - tuf_repository.bump_role_version = pretend.call_recorder( - lambda *a, **kw: fake_targets_md - ) - tuf_repository.snapshot_update_meta = pretend.call_recorder( - lambda *a, **kw: fake_snapshot_md - ) - tuf_repository._set_expiration_for_role = pretend.call_recorder( - lambda *a: fake_time - ) - result = tuf_repository.delegate_targets_roles(payload) - assert result == fake_snapshot_md - assert tuf_repository.load_role.calls == [ - pretend.call("snapshot"), - pretend.call("xxxx-yyyy"), - ] - assert tuf_repository._create_delegated_targets_roles.calls == [ - pretend.call(fake_targets_md, payload["xxxx-yyyy"], fake_snapshot_md) - ] - assert tuf_repository.bump_role_version.calls == [ - pretend.call( - rolename="xxxx-yyyy", - role_metadata=fake_targets_md, - role_expires=fake_time, - signers=None, - store=True, - ) - ] - assert tuf_repository.snapshot_update_meta.calls == [ - pretend.call("xxxx-yyyy", 2, fake_snapshot_md) - ] - assert tuf_repository._set_expiration_for_role.calls == [ - pretend.call("xxxx-yyyy") - ] - - def test_bump_role_version(self, tuf_repository): - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_new_time = datetime.datetime(2022, 6, 16, 9, 5, 1) - fake_signers = [ - pretend.stub( - key_dict={"keyid": "fake_id"}, - sign=pretend.call_recorder(lambda *a: "key1"), - ) - ] - initial_version = 1983 - fake_role_metadata = pretend.stub( - signed=pretend.stub(expires=fake_time, version=initial_version), - sign=lambda *a, **kw: None, - ) - - tuf_repository.key_backend = pretend.stub( - get=pretend.call_recorder(lambda role: fake_signers) - ) - - result = tuf_repository.bump_role_version( - "fake_role", fake_role_metadata, fake_new_time, fake_signers - ) - assert result.signed.version == initial_version + 1 - assert result.signed.expires == fake_new_time - - def test_bump_role_version_store_true(self, tuf_repository): - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_new_time = datetime.datetime(2022, 6, 16, 9, 5, 1) - initial_version = 1983 - fake_role_metadata = pretend.stub( - signed=pretend.stub(expires=fake_time, version=initial_version), - sign=lambda *a, **kw: None, - ) - fake_signers = [ - pretend.stub( - key_dict={"keyid": "fake_id"}, - sign=pretend.call_recorder(lambda *a: "key1"), - ) - ] - - tuf_repository._store = pretend.call_recorder(lambda rolename, role_md: None) - result = tuf_repository.bump_role_version( - "fake_role", fake_role_metadata, fake_new_time, fake_signers, store=True - ) - assert result.signed.version == initial_version + 1 - assert result.signed.expires == fake_new_time - assert tuf_repository._store.calls == [ - pretend.call("fake_role", fake_role_metadata) - ] - - def test_bump_role_version_with_key_rolename(self, tuf_repository): - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_new_time = datetime.datetime(2022, 6, 16, 9, 5, 1) - initial_version = 1983 - fake_role_metadata = pretend.stub( - signed=pretend.stub(expires=fake_time, version=initial_version), - sign=lambda *a, **kw: None, - ) - fake_signers = [ - pretend.stub( - key_dict={"keyid": "fake_id"}, - sign=pretend.call_recorder(lambda *a: "key1"), - ) - ] - - result = tuf_repository.bump_role_version( - "fake_role", fake_role_metadata, fake_new_time, fake_signers - ) - assert result.signed.version == initial_version + 1 - assert result.signed.expires == fake_new_time - - def test_bump_timestamp_version(self, tuf_repository): - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_new_time = datetime.datetime(2022, 6, 16, 9, 5, 1) - initial_version = 10 - fake_timestamp_md = pretend.stub( - signed=pretend.stub( - expires=fake_time, version=initial_version, snapshot_meta=1 - ), - sign=lambda *a, **kw: None, - ) - tuf_repository._set_expiration_for_role = pretend.call_recorder( - lambda role: fake_new_time - ) - tuf_repository.load_role = pretend.call_recorder(lambda role: fake_timestamp_md) - tuf_repository.key_backend = pretend.stub( - get=pretend.call_recorder(lambda role: [{"key": "key_data"}]) - ) - - result = tuf_repository.timestamp_bump_version(snapshot_version=20) - assert result.signed.version == initial_version + 1 - assert result.signed.expires == fake_new_time - assert tuf_repository.load_role.calls == [pretend.call("timestamp")] - assert tuf_repository.key_backend.get.calls == [pretend.call("timestamp")] - assert tuf_repository._set_expiration_for_role.calls == [ - pretend.call("timestamp") - ] - - def test_bump_timestamp_version_store_true(self, tuf_repository): - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_new_time = datetime.datetime(2022, 6, 16, 9, 5, 1) - initial_version = 10 - fake_timestamp_md = pretend.stub( - signed=pretend.stub( - expires=fake_time, version=initial_version, snapshot_meta=1 - ), - sign=lambda *a, **kw: None, - ) - - tuf_repository.load_role = pretend.call_recorder(lambda role: fake_timestamp_md) - tuf_repository.key_backend = pretend.stub( - get=pretend.call_recorder(lambda role: [{"key": "key_data"}]) - ) - tuf_repository._set_expiration_for_role = pretend.call_recorder( - lambda role: fake_new_time - ) - tuf_repository._store = pretend.call_recorder(lambda role, role_md: None) - - result = tuf_repository.timestamp_bump_version(snapshot_version=20, store=True) - assert result.signed.version == initial_version + 1 - assert result.signed.expires == fake_new_time - assert result.signed.snapshot_meta.version == 20 - assert tuf_repository.load_role.calls == [pretend.call("timestamp")] - assert tuf_repository.key_backend.get.calls == [pretend.call("timestamp")] - assert tuf_repository._set_expiration_for_role.calls == [ - pretend.call("timestamp") - ] - assert tuf_repository._store.calls == [ - pretend.call("timestamp", fake_timestamp_md) - ] - - def test_bump_snapshot_version(self, tuf_repository): - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_new_time = datetime.datetime(2022, 6, 16, 9, 5, 1) - initial_version = 10 - fake_snapshot_md = pretend.stub( - signed=pretend.stub( - expires=fake_time, version=initial_version, snapshot_meta=1 - ), - sign=lambda *a, **kw: None, - ) - - tuf_repository.load_role = pretend.call_recorder(lambda role: fake_snapshot_md) - tuf_repository._set_expiration_for_role = pretend.call_recorder( - lambda role: fake_new_time - ) - tuf_repository.key_backend = pretend.stub( - get=pretend.call_recorder(lambda role: [{"key": "key_data"}]) - ) - - result = tuf_repository.snapshot_bump_version() - assert result.signed.version == initial_version + 1 - assert result.signed.expires == fake_new_time - assert tuf_repository.load_role.calls == [pretend.call("snapshot")] - assert tuf_repository.key_backend.get.calls == [pretend.call("snapshot")] - assert tuf_repository._set_expiration_for_role.calls == [ - pretend.call("snapshot") - ] - - def test_bump_snapshot_version_store_true(self, tuf_repository): - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_new_time = datetime.datetime(2022, 6, 16, 9, 5, 1) - initial_version = 10 - fake_snapshot_md = pretend.stub( - signed=pretend.stub( - expires=fake_time, version=initial_version, snapshot_meta=1 - ), - sign=lambda *a, **kw: None, - ) - - tuf_repository.load_role = pretend.call_recorder(lambda role: fake_snapshot_md) - tuf_repository.key_backend = pretend.stub( - get=pretend.call_recorder(lambda role: [{"key": "key_data"}]) - ) - tuf_repository._set_expiration_for_role = pretend.call_recorder( - lambda role: fake_new_time - ) - tuf_repository._store = pretend.call_recorder(lambda role, role_md: None) - - result = tuf_repository.snapshot_bump_version(store=True) - assert result.signed.version == initial_version + 1 - assert result.signed.expires == fake_new_time - assert tuf_repository.load_role.calls == [pretend.call("snapshot")] - assert tuf_repository.key_backend.get.calls == [pretend.call("snapshot")] - assert tuf_repository._set_expiration_for_role.calls == [ - pretend.call("snapshot") - ] - assert tuf_repository._store.calls == [ - pretend.call("snapshot", fake_snapshot_md) - ] - - def test_bump_snapshot_version_with_snapshot_metadata(self, tuf_repository): - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_new_time = datetime.datetime(2022, 6, 16, 9, 5, 1) - initial_version = 10 - fake_snapshot_md = pretend.stub( - signed=pretend.stub( - expires=fake_time, version=initial_version, snapshot_meta=1 - ), - sign=lambda *a, **kw: None, - ) - - tuf_repository.key_backend = pretend.stub( - get=pretend.call_recorder(lambda role: [{"key": "key_data"}]) - ) - tuf_repository._set_expiration_for_role = pretend.call_recorder( - lambda role: fake_new_time - ) - - result = tuf_repository.snapshot_bump_version(fake_snapshot_md) - assert result.signed.version == initial_version + 1 - assert result.signed.expires == fake_new_time - assert tuf_repository.key_backend.get.calls == [pretend.call("snapshot")] - assert tuf_repository._set_expiration_for_role.calls == [ - pretend.call("snapshot") - ] - - def test_snapshot_update_meta(self, tuf_repository): - - fake_snapshot_md = pretend.stub( - signed=pretend.stub(meta={"fake_role.json": MetaFile(version=2)}), - sign=lambda *a, **kw: None, - ) - - tuf_repository.load_role = pretend.call_recorder(lambda role: fake_snapshot_md) - - result = tuf_repository.snapshot_update_meta("fake_role", 3) - - assert result.signed.meta["fake_role.json"].version == 3 - assert tuf_repository.load_role.calls == [pretend.call("snapshot")] - - def test_add_targets(self, tuf_repository): - - payload = { - "a0-07": [ - repository.TargetFile.from_dict( - { - "hashes": {"blake2b-256": "sdfaslkajsdfkjhadsljkhfsdjkh"}, - "length": 1024, - "custom": {"backsigned": True}, - }, - "/sd/fa/lkajsdfkjhadsljkhfsdjkh.packagexv1.tar.gz", - ), - repository.TargetFile.from_dict( - { - "hashes": {"blake2b-256": "dlskjflkdjflsdjfsdfdfsdfsdfs"}, - "length": 1025, - "custom": {"backsigned": True}, - }, - "/sd/fa/dlskjflkdjflsdjfsdfdfsdfsdfs.packageyv1.tar.gz", - ), - ] - } - - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - initial_version = 5 - - fake_snapshot_md = pretend.stub( - signed=pretend.stub( - expires=fake_time, version=initial_version, snapshot_meta=1 - ), - sign=lambda *a, **kw: None, - ) - fake_role_metadata = pretend.stub( - signed=pretend.stub(targets={}, version=initial_version, expires=fake_time), - sign=lambda *a, **kw: None, - ) - - tuf_repository.load_role = pretend.call_recorder( - lambda role: fake_snapshot_md - if role == Snapshot.type - else fake_role_metadata - ) - tuf_repository.key_backend = pretend.stub( - get=pretend.call_recorder(lambda role: [{"key": "key_data"}]) - ) - tuf_repository._store = pretend.call_recorder(lambda role, role_md: None) - tuf_repository.bump_role_version = pretend.call_recorder( - lambda *a, **kw: fake_role_metadata - ) - tuf_repository.snapshot_update_meta = pretend.call_recorder( - lambda *a, **kw: fake_snapshot_md - ) - - result = tuf_repository.add_targets(payload, "bins") - assert result == fake_snapshot_md - assert fake_role_metadata.signed.version == initial_version + 1 - assert tuf_repository.load_role.calls == [ - pretend.call("snapshot"), - pretend.call("a0-07"), - ] - assert tuf_repository.key_backend.get.calls == [pretend.call("bins")] - assert tuf_repository._store.calls == [ - pretend.call("a0-07", fake_role_metadata) - ] - assert tuf_repository.snapshot_update_meta.calls == [ - pretend.call("a0-07", initial_version + 1, fake_snapshot_md) - ] diff --git a/tests/unit/tuf/test_services.py b/tests/unit/tuf/test_services.py index 8508d7bd2d29..5b661c82785f 100644 --- a/tests/unit/tuf/test_services.py +++ b/tests/unit/tuf/test_services.py @@ -23,7 +23,6 @@ from warehouse.config import Environment from warehouse.tuf import services from warehouse.tuf.constants import BIN_N_COUNT, Role -from warehouse.tuf.hash_bins import HashBins from warehouse.tuf.interfaces import IKeyService, IRepositoryService, IStorageService @@ -166,7 +165,7 @@ def test_get_timestamp_specific(self, monkeypatch): services.__builtins__, "open", lambda *a, **kw: fake_file_object ) - with service.get("timestamp") as r: + with service.get(Role.TIMESTAMP.value) as r: result = r.read() assert result == fake_file_object.read() @@ -295,7 +294,15 @@ class TestRepositoryService: def test_verify_service(self): assert verifyClass(IRepositoryService, services.RepositoryService) - def test_create_service(self, db_request): + def test_basic_init(self): + service = services.RepositoryService( + "fake_storage", "fake_key_storage", "fake_request" + ) + assert service._storage_backend == "fake_storage" + assert service._key_storage_backend == "fake_key_storage" + assert service._request == "fake_request" + + def test_create_service(self): fake_service = "Fake Service" request = pretend.stub( find_service=pretend.call_recorder(lambda interface: fake_service) @@ -309,47 +316,154 @@ def test_create_service(self, db_request): pretend.call(IKeyService), ] - def test_basic_init(self): - service = services.RepositoryService( - "fake_storage", "fake_key_storage", "fake_request" + def test__get_bit_lenght(self, db_request): + db_request.registry.settings["warehouse.env"] = Environment.development + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request ) - assert service._storage_backend == "fake_storage" - assert service._key_storage_backend == "fake_key_storage" - assert service._request == "fake_request" + response = repository_service._get_bit_length() + assert response == 8 - def test__get_hash_bins_development(self, db_request): - fake_storage = pretend.stub() - fake_key_storage = pretend.stub() - db_request.registry.settings["warehouse.env"] = Environment.development - service = services.RepositoryService(fake_storage, fake_key_storage, db_request) + def test__get_bit_lenght_production(self, db_request): + db_request.registry.settings["warehouse.env"] = Environment.production + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() - result = service._get_hash_bins() - assert type(result) == HashBins - assert result.number_of_bins == 32 - assert result.number_of_prefixes == 256 - assert result.bin_size == 8 + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + response = repository_service._get_bit_length() + assert response == BIN_N_COUNT - def test__get_hash_bins_production(self, db_request): - fake_storage = pretend.stub() - fake_key_storage = pretend.stub() - db_request.registry.settings["warehouse.env"] = Environment.production - service = services.RepositoryService(fake_storage, fake_key_storage, db_request) + def test__is_initialized_true(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + repository_service._load = pretend.call_recorder(lambda *a: services.Root()) - result = service._get_hash_bins() - assert type(result) == HashBins - assert result.number_of_bins == BIN_N_COUNT - assert result.number_of_prefixes == 65536 - assert result.bin_size == 4 + assert repository_service._is_initialized() is True + assert repository_service._load.calls in [ + [pretend.call(role)] for role in services.TOP_LEVEL_ROLE_NAMES + ] + + def test__is_initialized_false(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() - def test_init_repository(self, db_request, monkeypatch): - fake_storage = pretend.stub() - fake_key_storage = pretend.stub( - get=pretend.call_recorder(lambda role: "fake_key") + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request ) + repository_service._load = pretend.call_recorder(lambda *a: None) - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_datetime = pretend.stub(now=pretend.call_recorder(lambda: fake_time)) - monkeypatch.setattr(datetime, "datetime", fake_datetime) + assert repository_service._is_initialized() is False + for pretend_call in repository_service._load.calls: + assert pretend_call in [ + pretend.call(role) for role in services.TOP_LEVEL_ROLE_NAMES + ] + + def test__is_initialized_false_by_exception(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + repository_service._load = pretend.raiser(services.StorageError) + + assert repository_service._is_initialized() is False + + def test__load(self, monkeypatch, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + fake_metadata = pretend.stub( + from_file=pretend.call_recorder(lambda *a: "Metadata") + ) + monkeypatch.setattr("warehouse.tuf.services.Metadata", fake_metadata) + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + result = repository_service._load("root") + + assert result == "Metadata" + assert fake_metadata.from_file.calls == [ + pretend.call("root", None, fake_storage_service) + ] + + def test__sign(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub( + get=pretend.call_recorder(lambda *a: ["signer1"]) + ) + + role = pretend.stub( + signatures=pretend.stub(clear=pretend.call_recorder(lambda: None)), + sign=pretend.call_recorder(lambda *a, **kw: None), + ) + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + result = repository_service._sign(role, "fake_role") + + assert result is None + assert fake_key_service.get.calls == [pretend.call("fake_role")] + assert role.signatures.clear.calls == [pretend.call()] + assert role.sign.calls == [pretend.call("signer1", append=True)] + + def test__persist(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + role = pretend.stub( + signed=pretend.stub(version=2), + to_file=pretend.call_recorder(lambda *a, **kw: None), + ) + + services.JSONSerializer = pretend.call_recorder(lambda: None) + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + result = repository_service._persist(role, "root") + + assert result is None + assert role.to_file.calls == [ + pretend.call("2.root.json", services.JSONSerializer(), fake_storage_service) + ] + assert services.JSONSerializer.calls == [pretend.call(), pretend.call()] + + def test__persist_timestamp(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + role = pretend.stub( + signed=pretend.stub(version=2), + to_file=pretend.call_recorder(lambda *a, **kw: None), + ) + + services.JSONSerializer = pretend.call_recorder(lambda: None) + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + result = repository_service._persist(role, Role.TIMESTAMP.value) + + assert result is None + assert role.to_file.calls == [ + pretend.call( + "timestamp.json", services.JSONSerializer(), fake_storage_service + ) + ] + assert services.JSONSerializer.calls == [pretend.call(), pretend.call()] + + def test__bump_expiry(self, monkeypatch, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() db_request.registry.settings["warehouse.env"] = Environment.production test_tuf_config = { @@ -365,324 +479,440 @@ def test_init_repository(self, db_request, monkeypatch): for name, value in test_tuf_config.items(): db_request.registry.settings[name] = value - fake_metadata_repository = pretend.stub( - is_initialized=False, - initialize=pretend.call_recorder(lambda *a, **kw: None), + fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) + fake_datetime = pretend.stub(now=pretend.call_recorder(lambda: fake_time)) + monkeypatch.setattr("warehouse.tuf.services.datetime", fake_datetime) + + role = pretend.stub( + signed=pretend.stub(expires=fake_datetime), ) - monkeypatch.setattr( - "warehouse.tuf.services.MetadataRepository", - lambda *a, **kw: fake_metadata_repository, + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request ) + result = repository_service._bump_expiry(role, "root") - service = services.RepositoryService(fake_storage, fake_key_storage, db_request) - result = service.init_dev_repository() + assert result is None + assert role.signed.expires == datetime.datetime(2020, 6, 15, 9, 5, 1) + assert fake_datetime.now.calls == [pretend.call()] + + def test__bump_version(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + role = pretend.stub( + signed=pretend.stub(version=2), + ) + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + result = repository_service._bump_version(role) assert result is None - assert fake_metadata_repository.initialize.calls == [ - pretend.call( - { - "targets": "fake_key", - "root": "fake_key", - "timestamp": "fake_key", - "snapshot": "fake_key", - }, - store=True, - ) + assert role.signed.version == 3 + + def test__update_timestamp(self, monkeypatch, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + + snapshot_version = 3 + fake_metafile = pretend.call_recorder(lambda *a, **kw: snapshot_version) + monkeypatch.setattr("warehouse.tuf.services.MetaFile", fake_metafile) + + mocked_timestamp = pretend.stub(signed=pretend.stub(snapshot_meta=2)) + repository_service._load = pretend.call_recorder(lambda *a: mocked_timestamp) + repository_service._bump_version = pretend.call_recorder(lambda *a: None) + repository_service._bump_expiry = pretend.call_recorder(lambda *a: None) + repository_service._sign = pretend.call_recorder(lambda *a: None) + repository_service._persist = pretend.call_recorder(lambda *a: None) + + result = repository_service._update_timestamp(snapshot_version) + + assert result is None + assert mocked_timestamp.signed.snapshot_meta == snapshot_version + assert repository_service._load.calls == [pretend.call(Role.TIMESTAMP.value)] + assert repository_service._bump_version.calls == [ + pretend.call(mocked_timestamp) + ] + assert repository_service._bump_expiry.calls == [ + pretend.call(mocked_timestamp, Role.TIMESTAMP.value) + ] + assert repository_service._sign.calls == [ + pretend.call(mocked_timestamp, Role.TIMESTAMP.value) + ] + assert repository_service._persist.calls == [ + pretend.call(mocked_timestamp, Role.TIMESTAMP.value) ] - for test_call in [ - pretend.call(Role.SNAPSHOT.value), - pretend.call(Role.ROOT.value), - pretend.call(Role.TARGETS.value), - pretend.call(Role.TIMESTAMP.value), - ]: - assert test_call in fake_key_storage.get.calls - - def test_init_repository_already_initialized(self, db_request, monkeypatch): - fake_storage = pretend.stub() - fake_key_storage = pretend.stub() - db_request.registry.settings["warehouse.env"] = Environment.production + def test__update_snapshot(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() - fake_metadata_repository = pretend.stub( - is_initialized=True, + snapshot_version = 3 + test_target_meta = [("bins", 3), ("f", 4)] + mocked_snapshot = pretend.stub( + signed=pretend.stub( + meta={}, + version=snapshot_version, + ) ) - monkeypatch.setattr( - "warehouse.tuf.services.MetadataRepository", - lambda *a, **kw: fake_metadata_repository, + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request ) - service = services.RepositoryService(fake_storage, fake_key_storage, db_request) + repository_service._load = pretend.call_recorder(lambda *a: mocked_snapshot) + repository_service._bump_version = pretend.call_recorder(lambda *a: None) + repository_service._bump_expiry = pretend.call_recorder(lambda *a: None) + repository_service._sign = pretend.call_recorder(lambda *a: None) + repository_service._persist = pretend.call_recorder(lambda *a: None) - with pytest.raises(FileExistsError) as err: - service.init_dev_repository() + result = repository_service._update_snapshot(test_target_meta) - assert "TUF Metadata Repository files already exists." in str(err.value) + assert result is snapshot_version + assert repository_service._load.calls == [pretend.call(Role.SNAPSHOT.value)] + assert repository_service._bump_version.calls == [pretend.call(mocked_snapshot)] + assert repository_service._bump_expiry.calls == [ + pretend.call(mocked_snapshot, Role.SNAPSHOT.value) + ] + assert repository_service._sign.calls == [ + pretend.call(mocked_snapshot, Role.SNAPSHOT.value) + ] + assert repository_service._persist.calls == [ + pretend.call(mocked_snapshot, Role.SNAPSHOT.value) + ] - def test_init_targets_delegation(self, db_request, monkeypatch): - fake_storage = pretend.stub() + def test_init_dev_repository(self, db_request): + fake_key = { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid": ( + "6dcd53f0a90fca17700f819e939a74b133aa5cd8619f3dc03228c0c68dcc2abb" + ), + "keyid_hash_algorithms": ["sha256", "sha512"], + "keyval": { + "public": ( + "c864d93b521d5851275a7b7c79fb0ac76311c206262eabd67319eba6665b1417" + ), + "private": ( + "bbe40143bfe1a3b6a41647f590e398fb8dd38fddf6b279edefdc022cdb649cdc" + ), + }, + } fake_signers = [ pretend.stub( - key_dict={"keyid": "fake_id"}, + key_dict=fake_key, sign=pretend.call_recorder(lambda *a: "key1"), - ) + ), + pretend.stub( + key_dict=fake_key, + sign=pretend.call_recorder(lambda *a: "key1"), + ), ] - fake_key_storage = pretend.stub( - get=pretend.call_recorder(lambda role: fake_signers) + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub( + get=pretend.call_recorder(lambda *a: fake_signers) ) - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_datetime = pretend.stub(now=pretend.call_recorder(lambda: fake_time)) - monkeypatch.setattr(datetime, "datetime", fake_datetime) - db_request.registry.settings["warehouse.env"] = Environment.production test_tuf_config = { + "tuf.root.threshold": 1, + "tuf.root.expiry": 31536000, "tuf.snapshot.threshold": 1, "tuf.snapshot.expiry": 86400, "tuf.targets.threshold": 2, "tuf.targets.expiry": 31536000, - "tuf.bins.threshold": 1, - "tuf.bins.expiry": 31536000, - "tuf.bin-n.threshold": 1, - "tuf.bin-n.expiry": 604800, + "tuf.timestamp.threshold": 1, + "tuf.timestamp.expiry": 86400, } for name, value in test_tuf_config.items(): db_request.registry.settings[name] = value - fake_metadata_repository = pretend.stub( - is_initialized=False, - delegate_targets_roles=pretend.call_recorder(lambda *a: None), - _set_expiration_for_role=pretend.call_recorder(lambda *a: fake_datetime), - ) - monkeypatch.setattr( - "warehouse.tuf.services.MetadataRepository", - lambda *a, **kw: fake_metadata_repository, + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request ) - service = services.RepositoryService(fake_storage, fake_key_storage, db_request) - service.bump_snapshot = pretend.call_recorder(lambda snapshot_metadata: None) - result = service.init_targets_delegation() + repository_service._is_initialized = pretend.call_recorder(lambda: False) + repository_service._bump_expiry = pretend.call_recorder(lambda *a: None) + repository_service._sign = pretend.call_recorder(lambda *a: None) + repository_service._persist = pretend.call_recorder(lambda *a: None) + result = repository_service.init_dev_repository() assert result is None - call_args = fake_metadata_repository.delegate_targets_roles.calls[0].args[0] - assert sorted(["targets", "bins"]) == sorted(list(call_args.keys())) - assert len(call_args["targets"]) == 1 - assert type(call_args["targets"][0][0]) == services.DelegatedRole - assert call_args["targets"][0][1][0].key_dict == {"keyid": "fake_id"} - assert ( - len(call_args["bins"]) == 16384 - ) # PEP458 https://peps.python.org/pep-0458/#metadata-scalability - assert type(call_args["bins"][0][0]) == services.DelegatedRole - assert call_args["bins"][0][1][0].key_dict == {"keyid": "fake_id"} - # 1 target + # PEP458 https://peps.python.org/pep-0458/#metadata-scalability - assert len(fake_metadata_repository._set_expiration_for_role.calls) == 16385 - - def test_bump_snapshot(self, db_request, monkeypatch): - fake_storage = pretend.stub() - fake_key_storage = pretend.stub( - get=pretend.call_recorder(lambda role: "fake_key") - ) - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_datetime = pretend.stub(now=pretend.call_recorder(lambda: fake_time)) - monkeypatch.setattr(datetime, "datetime", fake_datetime) + def test_init_dev_repository_already_initialized(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() - db_request.registry.settings["warehouse.env"] = Environment.production - test_tuf_config = { - "tuf.snapshot.expiry": 86400, - "tuf.timestamp.expiry": 86400, - } - - for name, value in test_tuf_config.items(): - db_request.registry.settings[name] = value - - fake_snapshot = pretend.stub(signed=pretend.stub(version=2)) - fake_metadata_repository = pretend.stub( - load_role=pretend.call_recorder(lambda role: fake_snapshot), - snapshot_bump_version=pretend.call_recorder(lambda *a, **kw: fake_snapshot), - timestamp_bump_version=pretend.call_recorder(lambda *a, **kw: None), - ) - monkeypatch.setattr( - "warehouse.tuf.services.MetadataRepository", - lambda *a, **kw: fake_metadata_repository, + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request ) + repository_service._is_initialized = pretend.call_recorder(lambda: True) + with pytest.raises(FileExistsError) as err: + repository_service.init_dev_repository() - service = services.RepositoryService(fake_storage, fake_key_storage, db_request) - result = service.bump_snapshot() - - bump_s_calls = fake_metadata_repository.snapshot_bump_version.calls[0].kwargs - bump_t_calls = fake_metadata_repository.timestamp_bump_version.calls[0].kwargs - - assert result is None - assert fake_metadata_repository.load_role.calls == [pretend.call("snapshot")] - assert bump_s_calls["snapshot_metadata"].signed.version == 2 - assert bump_s_calls["store"] is True - assert bump_t_calls["snapshot_version"] == 2 - assert bump_t_calls["store"] is True + assert "TUF Metadata Repository files already exists." in str(err) - def test_bump_snapshot_specific_snapshot_metadata(self, db_request, monkeypatch): - fake_storage = pretend.stub() - fake_key_storage = pretend.stub( - get=pretend.call_recorder(lambda role: "fake_key") + def test_init_targets_delegation(self, db_request): + fake_key = { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid": ( + "6dcd53f0a90fca17700f819e939a74b133aa5cd8619f3dc03228c0c68dcc2abb" + ), + "keyid_hash_algorithms": ["sha256", "sha512"], + "keyval": { + "public": ( + "c864d93b521d5851275a7b7c79fb0ac76311c206262eabd67319eba6665b1417" + ), + "private": ( + "bbe40143bfe1a3b6a41647f590e398fb8dd38fddf6b279edefdc022cdb649cdc" + ), + }, + } + fake_signers = [ + pretend.stub( + key_dict=fake_key, + sign=pretend.call_recorder(lambda *a: "key1"), + ), + pretend.stub( + key_dict=fake_key, + sign=pretend.call_recorder(lambda *a: "key1"), + ), + ] + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub( + get=pretend.call_recorder(lambda *a: fake_signers) ) - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_datetime = pretend.stub(now=pretend.call_recorder(lambda: fake_time)) - monkeypatch.setattr(datetime, "datetime", fake_datetime) + db_request.registry.settings["warehouse.env"] = Environment.development - db_request.registry.settings["warehouse.env"] = Environment.production test_tuf_config = { + "tuf.root.threshold": 1, + "tuf.root.expiry": 31536000, + "tuf.snapshot.threshold": 1, "tuf.snapshot.expiry": 86400, + "tuf.targets.threshold": 2, + "tuf.targets.expiry": 31536000, + "tuf.timestamp.threshold": 1, "tuf.timestamp.expiry": 86400, + "tuf.bins.threshold": 1, + "tuf.bins.expiry": 31536000, + "tuf.bin-n.threshold": 1, + "tuf.bin-n.expiry": 604800, } - for name, value in test_tuf_config.items(): db_request.registry.settings[name] = value - fake_snapshot = pretend.stub(signed=pretend.stub(version=2)) - fake_metadata_repository = pretend.stub( - snapshot_bump_version=pretend.call_recorder(lambda *a, **kw: fake_snapshot), - timestamp_bump_version=pretend.call_recorder(lambda *a, **kw: None), + fake_targets = pretend.stub( + signed=pretend.stub( + delegations=None, + roles={}, + add_key=pretend.call_recorder(lambda *a: None), + version=3, + ) ) - monkeypatch.setattr( - "warehouse.tuf.services.MetadataRepository", - lambda *a, **kw: fake_metadata_repository, + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request ) + repository_service._load = pretend.call_recorder(lambda *a: fake_targets) + repository_service._bump_version = pretend.call_recorder(lambda *a: None) + repository_service._bump_expiry = pretend.call_recorder(lambda *a: None) + repository_service._sign = pretend.call_recorder(lambda *a: None) + repository_service._persist = pretend.call_recorder(lambda *a: None) + repository_service._update_timestamp = pretend.call_recorder(lambda *a: None) + repository_service._update_snapshot = pretend.call_recorder(lambda *a: 3) - service = services.RepositoryService(fake_storage, fake_key_storage, db_request) - result = service.bump_snapshot(fake_snapshot) - - bump_s_calls = fake_metadata_repository.snapshot_bump_version.calls[0].kwargs - bump_t_calls = fake_metadata_repository.timestamp_bump_version.calls[0].kwargs + result = repository_service.init_targets_delegation() assert result is None - assert bump_s_calls["snapshot_metadata"].signed.version == 2 - assert bump_s_calls["store"] is True - assert bump_t_calls["snapshot_version"] == 2 - assert bump_t_calls["store"] is True + assert repository_service._load.calls == [pretend.call("targets")] + assert repository_service._bump_version.calls == [pretend.call(fake_targets)] + assert repository_service._update_snapshot.calls == [ + pretend.call([("targets", 3), ("bins", 1)]) + ] + assert repository_service._update_timestamp.calls == [pretend.call(3)] - def test_bump_bin_n_roles(self, db_request, monkeypatch): - fake_storage = pretend.stub() + def test_add_hashed_targets(self, db_request): + fake_key = { + "keytype": "ed25519", + "scheme": "ed25519", + "keyid": ( + "6dcd53f0a90fca17700f819e939a74b133aa5cd8619f3dc03228c0c68dcc2abb" + ), + "keyid_hash_algorithms": ["sha256", "sha512"], + "keyval": { + "public": ( + "c864d93b521d5851275a7b7c79fb0ac76311c206262eabd67319eba6665b1417" + ), + "private": ( + "bbe40143bfe1a3b6a41647f590e398fb8dd38fddf6b279edefdc022cdb649cdc" + ), + }, + } fake_signers = [ pretend.stub( - key_dict={"keyid": "fake_id"}, + key_dict=fake_key, sign=pretend.call_recorder(lambda *a: "key1"), - ) + ), + pretend.stub( + key_dict=fake_key, + sign=pretend.call_recorder(lambda *a: "key1"), + ), ] - fake_key_storage = pretend.stub( - get=pretend.call_recorder(lambda role: fake_signers) + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub( + get=pretend.call_recorder(lambda *a: fake_signers) ) - fake_time = datetime.datetime(2019, 6, 16, 9, 5, 1) - fake_datetime = pretend.stub(now=pretend.call_recorder(lambda: fake_time)) - monkeypatch.setattr(datetime, "datetime", fake_datetime) + db_request.registry.settings["warehouse.env"] = Environment.development - db_request.registry.settings["warehouse.env"] = Environment.production test_tuf_config = { - "tuf.snapshot.expiry": 86400, - "tuf.timestamp.expiry": 86400, + "tuf.bin-n.threshold": 1, "tuf.bin-n.expiry": 604800, - "tuf.bins.expiry": 31536000, } - for name, value in test_tuf_config.items(): db_request.registry.settings[name] = value - fake_bin_n_metadata = pretend.stub(signed=pretend.stub(version=5)) - fake_snapshot = pretend.stub(signed=pretend.stub(version=2)) - fake_metadata_repository = pretend.stub( - load_role=pretend.call_recorder( - lambda role: fake_snapshot - if role == Role.SNAPSHOT.value - else fake_bin_n_metadata - ), - bump_role_version=pretend.call_recorder(lambda *a, **kw: None), - snapshot_bump_version=pretend.call_recorder(lambda *a, **kw: fake_snapshot), - snapshot_update_meta=pretend.call_recorder( - lambda *a, **kw: "snapshot_metadata" - ), - timestamp_bump_version=pretend.call_recorder(lambda *a, **kw: None), - _set_expiration_for_role=pretend.call_recorder(lambda *a: fake_datetime), - _key_storage_backend=pretend.call_recorder(lambda *a: fake_signers), + fake_bins = pretend.stub( + signed=pretend.stub( + delegations=pretend.stub( + succinct_roles=pretend.stub( + get_role_for_target=pretend.call_recorder(lambda *a: "bin-n-3d") + ) + ), + ) ) - monkeypatch.setattr( - "warehouse.tuf.services.MetadataRepository", - lambda *a, **kw: fake_metadata_repository, + fake_bin_n = pretend.stub(signed=pretend.stub(targets={}, version=4)) + + def mocked_load(role): + if role == "bins": + return fake_bins + else: + return fake_bin_n + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request ) - service = services.RepositoryService(fake_storage, fake_key_storage, db_request) - service.bump_snapshot = pretend.call_recorder(lambda snapshot_metadata: None) + repository_service._load = pretend.call_recorder(lambda r: mocked_load(r)) + repository_service._bump_version = pretend.call_recorder(lambda *a: None) + repository_service._bump_expiry = pretend.call_recorder(lambda *a: None) + repository_service._sign = pretend.call_recorder(lambda *a: None) + repository_service._persist = pretend.call_recorder(lambda *a: None) + repository_service._update_timestamp = pretend.call_recorder(lambda *a: None) + repository_service._update_snapshot = pretend.call_recorder(lambda *a: 3) - result = service.bump_bin_n_roles() + targets = [ + services.TargetFile( + 1024, + {"blake2b-256": "fake_hash_0123456789abcdef"}, + "/xy/some_package.tar.gz", + {"backsigned": True}, + ), + services.TargetFile( + 1024, + {"blake2b-256": "fake_hash_0123456789abcdef"}, + "/xy/some_package.tar.gz", + {"backsigned": True}, + ), + ] + result = repository_service.add_hashed_targets(targets) assert result is None - # PEP458 https://peps.python.org/pep-0458/#metadata-scalability - assert len(fake_metadata_repository.bump_role_version.calls) == 16384 - assert len(fake_metadata_repository.snapshot_update_meta.calls) == 16384 - assert ( - len(fake_metadata_repository.load_role.calls) == 16385 - ) # +1 snapshot call - assert ( - fake_metadata_repository.load_role.calls.count(pretend.call("snapshot")) - == 1 - ) - assert len(fake_metadata_repository._set_expiration_for_role.calls) == 16384 - assert service.bump_snapshot.calls == [pretend.call("snapshot_metadata")] - - def test_add_hashed_targets(self, db_request, monkeypatch): - db_request.registry.settings["warehouse.env"] = Environment.production + assert repository_service._load.calls == [ + pretend.call("bins"), + pretend.call("bin-n-3d"), + ] + assert repository_service._bump_version.calls == [pretend.call(fake_bin_n)] + assert repository_service._bump_expiry.calls == [ + pretend.call(fake_bin_n, "bin-n") + ] + assert repository_service._sign.calls == [pretend.call(fake_bin_n, "bin-n")] + assert repository_service._sign.calls == [pretend.call(fake_bin_n, "bin-n")] + assert repository_service._update_snapshot.calls == [ + pretend.call([("bin-n-3d", 4)]) + ] + assert repository_service._update_timestamp.calls == [pretend.call(3)] - fake_storage = pretend.stub() - fake_key_storage = pretend.stub() - fake_metadata_repository = pretend.stub( - add_targets=pretend.call_recorder(lambda *a, **kw: "snapshot_metadata"), - ) - monkeypatch.setattr( - "warehouse.tuf.services.MetadataRepository", - lambda *a, **kw: fake_metadata_repository, - ) + def test_bump_bin_n_roles(self, db_request): - monkeypatch.setattr( - "warehouse.tuf.services.TargetFile.from_dict", - lambda *a, **kw: "target_dict", + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + fake_bins = pretend.stub( + signed=pretend.stub( + delegations=pretend.stub( + succinct_roles=pretend.stub( + get_roles=pretend.call_recorder(lambda: ["bin-0", "bin-f"]) + ) + ), + ) ) + fake_bin_n = pretend.stub(signed=pretend.stub(targets={}, version=5)) + + def mocked_load(role): + if role == "bins": + return fake_bins + else: + return fake_bin_n - fake_hash_bins = pretend.stub( - get_delegate=pretend.call_recorder(lambda filepath: "xxxx-yyyy") + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request ) - service = services.RepositoryService(fake_storage, fake_key_storage, db_request) - service.bump_snapshot = pretend.call_recorder(lambda snapshot_metadata: None) - service._get_hash_bins = pretend.call_recorder(lambda: fake_hash_bins) + repository_service._load = pretend.call_recorder(lambda r: mocked_load(r)) + repository_service._bump_version = pretend.call_recorder(lambda *a: None) + repository_service._bump_expiry = pretend.call_recorder(lambda *a: None) + repository_service._sign = pretend.call_recorder(lambda *a: None) + repository_service._persist = pretend.call_recorder(lambda *a: None) + repository_service._update_timestamp = pretend.call_recorder(lambda *a: None) + repository_service._update_snapshot = pretend.call_recorder(lambda *a: 6) - targets = [ - { - "info": { - "hashes": {"blake2b-256": "sdfaslkajsdfkjhadsljkhfsdjkh"}, - "length": 1024, - "custom": {"backsigned": True}, - }, - "path": "/sd/fa/lkajsdfkjhadsljkhfsdjkh.packagexv1.tar.gz", - }, - { - "info": { - "hashes": {"blake2b-256": "dlskjflkdjflsdjfsdfdfsdfsdfs"}, - "length": 1025, - "custom": {"backsigned": True}, - }, - "path": "/sd/fa/dlskjflkdjflsdjfsdfdfsdfsdfs.packageyv1.tar.gz", - }, - ] - result = service.add_hashed_targets(targets) + result = repository_service.bump_bin_n_roles() assert result is None - assert fake_metadata_repository.add_targets.calls == [ - pretend.call( - {"xxxx-yyyy": ["target_dict", "target_dict"]}, - "bin-n", - ) - ], fake_metadata_repository.add_targets.calls - assert service.bump_snapshot.calls == [pretend.call("snapshot_metadata")] - assert service._get_hash_bins.calls == [pretend.call()] - assert fake_hash_bins.get_delegate.calls == [ - pretend.call("/sd/fa/lkajsdfkjhadsljkhfsdjkh.packagexv1.tar.gz"), - pretend.call("/sd/fa/dlskjflkdjflsdjfsdfdfsdfsdfs.packageyv1.tar.gz"), + assert repository_service._load.calls == [ + pretend.call("bins"), + pretend.call("bin-0"), + pretend.call("bin-f"), + ] + assert repository_service._bump_version.calls == [ + pretend.call(fake_bin_n), + pretend.call(fake_bin_n), + ] + assert repository_service._bump_expiry.calls == [ + pretend.call(fake_bin_n, "bin-n"), + pretend.call(fake_bin_n, "bin-n"), + ] + assert repository_service._sign.calls == [ + pretend.call(fake_bin_n, "bin-n"), + pretend.call(fake_bin_n, "bin-n"), + ] + assert repository_service._sign.calls == [ + pretend.call(fake_bin_n, "bin-n"), + pretend.call(fake_bin_n, "bin-n"), ] + assert repository_service._update_snapshot.calls == [ + pretend.call([("bin-0", 5), ("bin-f", 5)]) + ] + assert repository_service._update_timestamp.calls == [pretend.call(6)] + + def test_bump_snapshot(self, db_request): + fake_storage_service = pretend.stub() + fake_key_service = pretend.stub() + + repository_service = services.RepositoryService( + fake_storage_service, fake_key_service, db_request + ) + repository_service._update_snapshot = pretend.call_recorder(lambda *a: 41) + repository_service._update_timestamp = pretend.call_recorder(lambda *a: None) + + result = repository_service.bump_snapshot() + + assert result is None + assert repository_service._update_snapshot.calls == [pretend.call([])] + assert repository_service._update_timestamp.calls == [pretend.call(41)] diff --git a/tests/unit/tuf/test_tasks.py b/tests/unit/tuf/test_tasks.py index 250fab5f7820..35e756d63fd6 100644 --- a/tests/unit/tuf/test_tasks.py +++ b/tests/unit/tuf/test_tasks.py @@ -14,7 +14,7 @@ from warehouse.tuf import tasks from warehouse.tuf.interfaces import IRepositoryService -from warehouse.tuf.repository import TargetFile +from warehouse.tuf.services import TargetFile class TestBumpSnapshot: diff --git a/warehouse/cli/tuf.py b/warehouse/cli/tuf.py index 4812eeeed491..7edf21be84c5 100644 --- a/warehouse/cli/tuf.py +++ b/warehouse/cli/tuf.py @@ -20,6 +20,7 @@ from warehouse.cli import warehouse from warehouse.packaging.utils import render_simple_detail from warehouse.tuf.tasks import ( + TargetFile, add_hashed_targets as _add_hashed_targets, bump_bin_n_roles as _bump_bin_n_roles, bump_snapshot as _bump_snapshot, @@ -135,11 +136,14 @@ def add_all_packages(config): targets = list() for file in db.query(File).all(): hashes = {"blake2b-256": file.blake2_256_digest} - targetinfo = dict() - targetinfo["length"] = file.size - targetinfo["hashes"] = hashes - targetinfo["custom"] = {"backsigned": True} - targets.append({"info": targetinfo, "path": file.path}) + targets.append( + TargetFile( + length=file.size, + hashes=hashes, + path=file.path, + unrecognized_fields={"backsigned": True}, + ) + ) config.task(_add_hashed_targets).run(request, targets) @@ -163,15 +167,12 @@ def add_all_indexes(config): except OSError as err: click.ClickException(str(err)) hashes = {"blake2b-256": simple_detail.get("content_hash")} - targetinfo = dict() - targetinfo["hashes"] = hashes - targetinfo["length"] = simple_detail.get("length") - targetinfo["custom"] = {"backsigned": True} targets.append( - { - "info": targetinfo, - "path": f"{project.normalized_name}/{project.normalized_name}.html", - } + TargetFile( + length=simple_detail.get("length"), + hashes=hashes, + path=f"{project.normalized_name}/{project.normalized_name}.html", + ) ) config.task(_add_hashed_targets).run(request, targets) diff --git a/warehouse/tuf/constants.py b/warehouse/tuf/constants.py index 3ed4c8fbfa05..7e37876f769c 100644 --- a/warehouse/tuf/constants.py +++ b/warehouse/tuf/constants.py @@ -27,4 +27,4 @@ class Role(enum.Enum): TUF_REPO_LOCK = "tuf-repo" -BIN_N_COUNT = 16384 +BIN_N_COUNT = 32 diff --git a/warehouse/tuf/hash_bins.py b/warehouse/tuf/hash_bins.py deleted file mode 100644 index 9e7fb43de251..000000000000 --- a/warehouse/tuf/hash_bins.py +++ /dev/null @@ -1,90 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import hashlib - -from typing import Iterator, List, Tuple - - -class HashBins: - """ - Hash Bins management - - This code is inspired on python-tuf repo examples for hash bins - """ - - def __init__(self, number_of_bins: int) -> None: - """ - Hash Bins - - Args: - number_of_bins: number of bins - """ - self.number_of_bins = number_of_bins - # The prefix length is the number of digits in the hexadecimal representation - # (see 'x' in Python Format Specification) of the number of bins minus one - # (counting starts at zero), i.e. ... - self.prefix_len = len(f"{(self.number_of_bins - 1):x}") # ... 2. - # Compared to decimal, hexadecimal numbers can express higher numbers - # with fewer digits and thus further decrease metadata sizes. With the - # above prefix length of 2 we can represent at most 256 prefixes, i.e. - # 00, 01, ..., ff. - self.number_of_prefixes = 16**self.prefix_len - # If the number of bins is a power of two, hash prefixes are evenly - # distributed over all bins, which allows to calculate the uniform size - # of 8, where each bin is responsible for a range of 8 prefixes, i.e. - # 00-07, 08-0f, ..., f8-ff. - self.bin_size = self.number_of_prefixes // self.number_of_bins - - def _bin_name(self, low: int, high: int) -> str: - """ - Generates a bin name according to the hash prefixes the bin serves. - - The name is either a single hash prefix for bin size 1, or a range of hash - prefixes otherwise. The prefix length is needed to zero-left-pad the - hex representation of the hash prefix for uniform bin name lengths. - """ - if low == high: - return f"{low:0{self.prefix_len}x}" - - return f"{low:0{self.prefix_len}x}-{high:0{self.prefix_len}x}" - - def generate(self) -> Iterator[Tuple[str, List[str]]]: - """Returns generator for bin names and hash prefixes per bin.""" - # Iterate over the total number of hash prefixes in 'bin size'-steps to - # generate bin names and a list of hash prefixes served by each bin. - for low in range(0, self.number_of_prefixes, self.bin_size): - high = low + self.bin_size - 1 - bin_name = self._bin_name(low, high) - hash_prefixes = [] - for prefix in range(low, low + self.bin_size): - hash_prefixes.append(f"{prefix:0{self.prefix_len}x}") - - yield bin_name, hash_prefixes - - def get_delegate(self, file_path: str) -> str: - """ - Gets the delegated role name bin based on the target file path. - - Args: - file_path - - Returns: - bin name low-high - """ - hasher = hashlib.sha256() - hasher.update(file_path.encode("utf-8")) - target_name_hash = hasher.hexdigest() - prefix = int(target_name_hash[: self.prefix_len], 16) - low = prefix - (prefix % self.bin_size) - high = low + self.bin_size - 1 - return self._bin_name(low, high) diff --git a/warehouse/tuf/services.py b/warehouse/tuf/services.py index c418024adea0..753d3fada153 100644 --- a/warehouse/tuf/services.py +++ b/warehouse/tuf/services.py @@ -18,7 +18,7 @@ from contextlib import contextmanager from datetime import datetime, timedelta -from typing import List, Optional, Tuple +from typing import Dict, List, Tuple from securesystemslib.exceptions import StorageError # type: ignore from securesystemslib.interface import ( # type: ignore @@ -36,6 +36,7 @@ Role, Root, Snapshot, + SuccinctRoles, TargetFile, Targets, Timestamp, @@ -45,7 +46,6 @@ from warehouse.config import Environment from warehouse.tuf.constants import BIN_N_COUNT, Role as RoleType -from warehouse.tuf.hash_bins import HashBins from warehouse.tuf.interfaces import IKeyService, IRepositoryService, IStorageService SPEC_VERSION: str = ".".join(SPECIFICATION_VERSION) @@ -186,19 +186,21 @@ def create_service(cls, context, request): key_service = request.find_service(IKeyService) return cls(storage_service, key_service, request) - def _get_hash_bins(self): + def _get_bit_length(self): """ Returns a 'hash bin delegation' management object. """ if self._request.registry.settings["warehouse.env"] == Environment.development: - number_of_bins = 32 + bit_length = 8 else: - number_of_bins = BIN_N_COUNT + bit_length = BIN_N_COUNT - return HashBins(number_of_bins) + return bit_length def _is_initialized(self) -> bool: - """Returns True if any top-level role metadata exists, False otherwise.""" + """ + Returns True if any top-level role metadata exists, False otherwise. + """ try: if any(role for role in TOP_LEVEL_ROLE_NAMES if self._load(role)): return True @@ -217,7 +219,8 @@ def _load(self, role_name: str) -> Metadata: return Metadata.from_file(role_name, None, self._storage_backend) def _sign(self, role: Metadata, role_name: str) -> None: - """Re-signs metadata with role-specific key from global key store. + """ + Re-signs metadata with role-specific key from global key store. The metadata role type is used as default key id. This is only allowed for top-level roles. @@ -227,7 +230,8 @@ def _sign(self, role: Metadata, role_name: str) -> None: role.sign(signer, append=True) def _persist(self, role: Metadata, role_name: str) -> None: - """Persists metadata using the configured storage backend. + """ + Persists metadata using the configured storage backend. The metadata role type is used as default role name. This is only allowed for top-level roles. All names but 'timestamp' are prefixed with a version number. @@ -240,7 +244,8 @@ def _persist(self, role: Metadata, role_name: str) -> None: role.to_file(filename, JSONSerializer(), self._storage_backend) def _bump_expiry(self, role: Metadata, expiry_id: str) -> None: - """Bumps metadata expiration date by role-specific interval. + """ + Bumps metadata expiration date by role-specific interval. The metadata role type is used as default expiry id. This is only allowed for top-level roles. @@ -255,12 +260,16 @@ def _bump_expiry(self, role: Metadata, expiry_id: str) -> None: ) def _bump_version(self, role: Metadata) -> None: - """Bumps metadata version by 1.""" + """ + Bumps metadata version by 1. + """ role.signed.version += 1 - def _update_timestamp(self, snapshot_version: int) -> Metadata[Timestamp]: - """Loads 'timestamp', updates meta info about passed 'snapshot' metadata, - bumps version and expiration, signs and persists.""" + def _update_timestamp(self, snapshot_version: int) -> None: + """ + Loads 'timestamp', updates meta info about passed 'snapshot' metadata, + bumps version and expiration, signs and persists. + """ timestamp = self._load(Timestamp.type) timestamp.signed.snapshot_meta = MetaFile(version=snapshot_version) @@ -269,12 +278,12 @@ def _update_timestamp(self, snapshot_version: int) -> Metadata[Timestamp]: self._sign(timestamp, RoleType.TIMESTAMP.value) self._persist(timestamp, RoleType.TIMESTAMP.value) - def _update_snapshot( - self, targets_meta: List[Tuple[str, int]] - ) -> Metadata[Snapshot]: - """Loads 'snapshot', updates meta info about passed 'targets' metadata, bumps + def _update_snapshot(self, targets_meta: List[Tuple[str, int]]) -> int: + """ + Loads 'snapshot', updates meta info about passed 'targets' metadata, bumps version and expiration, signs and persists. Returns new snapshot version, e.g. - to update 'timestamp'.""" + to update 'timestamp'. + """ snapshot = self._load(Snapshot.type) for name, version in targets_meta: @@ -287,7 +296,7 @@ def _update_snapshot( return snapshot.signed.version - def init_dev_repository(self): + def init_dev_repository(self) -> None: """ Creates development TUF top-level role metadata (root, targets, snapshot, timestamp). @@ -320,18 +329,18 @@ def init_dev_repository(self): f"signing threshold '{threshold}'" ) - root.roles[role_name] = Role([], threshold) + root.roles[role_name] = Role([], threshold) # type: ignore for signer in signers: - root.add_key(role_name, Key.from_securesystemslib_key(signer.key_dict)) + root.add_key(Key.from_securesystemslib_key(signer.key_dict), role_name) # Add signature wrapper, bump expiration, and sign and persist for role in [targets, snapshot, timestamp, root]: - metadata = Metadata(role) + metadata = Metadata(role) # type: ignore self._bump_expiry(metadata, role.type) self._sign(metadata, role.type) self._persist(metadata, role.type) - def init_targets_delegation(self): + def init_targets_delegation(self) -> None: """ Creates TUF metadata for hash bin delegated targets roles (bins, bin-n). @@ -355,7 +364,9 @@ def init_targets_delegation(self): # signature thresholds. targets = self._load(Targets.type) targets.signed.delegations = Delegations(keys={}, roles={}) - targets.signed.delegations.roles[RoleType.BINS.value] = DelegatedRole( + targets.signed.delegations.roles[ # type: ignore + RoleType.BINS.value + ] = DelegatedRole( name=RoleType.BINS.value, keyids=[], threshold=self._request.registry.settings[ @@ -367,7 +378,7 @@ def init_targets_delegation(self): for signer in self._key_storage_backend.get(RoleType.BINS.value): targets.signed.add_key( - RoleType.BINS.value, Key.from_securesystemslib_key(signer.key_dict) + Key.from_securesystemslib_key(signer.key_dict), RoleType.BINS.value ) # Bump version and expiration, and sign and persist updated 'targets'. @@ -378,36 +389,22 @@ def init_targets_delegation(self): targets_meta.append((RoleType.TARGETS.value, targets.signed.version)) + succinct_roles = SuccinctRoles( + [], 1, self._get_bit_length(), RoleType.BIN_N.value + ) # Create new 'bins' role and delegate trust from 'bins' for all target files to # 'bin-n' roles based on file path hash prefixes, a.k.a hash bin delegation. bins = Metadata(Targets()) - bins.signed.delegations = Delegations(keys={}, roles={}) - hash_bins = self._get_hash_bins() - for bin_n_name, bin_n_hash_prefixes in hash_bins.generate(): - bins.signed.delegations.roles[bin_n_name] = DelegatedRole( - name=bin_n_name, - keyids=[], - threshold=self._request.registry.settings[ - f"tuf.{RoleType.BIN_N.value}.threshold" - ], - terminating=False, - path_hash_prefixes=bin_n_hash_prefixes, - ) - + bins.signed.delegations = Delegations(keys={}, succinct_roles=succinct_roles) + for delegated_name in succinct_roles.get_roles(): for signer in self._key_storage_backend.get(RoleType.BIN_N.value): bins.signed.add_key( - bin_n_name, Key.from_securesystemslib_key(signer.key_dict) + Key.from_securesystemslib_key(signer.key_dict), delegated_name ) - - # Create new empty 'bin-n' roles, bump expiration, and sign and persist bin_n = Metadata(Targets()) self._bump_expiry(bin_n, RoleType.BIN_N.value) self._sign(bin_n, RoleType.BIN_N.value) - self._persist(bin_n, bin_n_name) - - # FIXME: Possible performance gain by updating 'snapshot' right here, to - # omit creation of massive list and iterating over all 'bin-n' roles twice. - targets_meta.append((bin_n_name, bin_n.signed.version)) + self._persist(bin_n, delegated_name) # Bump expiration, and sign and persist new 'bins' role. self._bump_expiry(bins, RoleType.BINS.value) @@ -418,7 +415,7 @@ def init_targets_delegation(self): self._update_timestamp(self._update_snapshot(targets_meta)) - def add_hashed_targets(self, targets): + def add_hashed_targets(self, targets: List[TargetFile]) -> None: """ Updates 'bin-n' roles metadata, assigning each passed target to the correct bin. @@ -428,16 +425,17 @@ def add_hashed_targets(self, targets): Updating 'bin-n' also updates 'snapshot' and 'timestamp'. """ # Group target files by responsible 'bin-n' roles - bin_n_target_groups = {} - hash_bins = self._get_hash_bins() + bin_n = self._load(RoleType.BINS.value) + bin_n_succinct_roles = bin_n.signed.delegations.succinct_roles + bin_n_target_groups: Dict[str, List[TargetFile]] = {} + for target in targets: - bin_n_name = hash_bins.get_delegate(target["path"]) + bin_n_name = bin_n_succinct_roles.get_role_for_target(target.path) if bin_n_name not in bin_n_target_groups: bin_n_target_groups[bin_n_name] = [] - target_file = TargetFile.from_dict(target["info"], target["path"]) - bin_n_target_groups[bin_n_name].append(target_file) + bin_n_target_groups[bin_n_name].append(target) # Update target file info in responsible 'bin-n' roles, bump version and expiry # and sign and persist @@ -457,7 +455,7 @@ def add_hashed_targets(self, targets): self._update_timestamp(self._update_snapshot(targets_meta)) - def bump_bin_n_roles(self): + def bump_bin_n_roles(self) -> None: """ Bumps version and expiration date of 'bin-n' role metadata (multiple). @@ -467,9 +465,10 @@ def bump_bin_n_roles(self): Updating 'bin-n' also updates 'snapshot' and 'timestamp'. """ - hash_bins = self._get_hash_bins() + bin_n = self._load(RoleType.BINS.value) + bin_n_succinct_roles = bin_n.signed.delegations.succinct_roles targets_meta = [] - for bin_n_name, _ in hash_bins.generate(): + for bin_n_name in bin_n_succinct_roles.get_roles(): bin_n = self._load(bin_n_name) self._bump_expiry(bin_n, RoleType.BIN_N.value) @@ -481,7 +480,7 @@ def bump_bin_n_roles(self): self._update_timestamp(self._update_snapshot(targets_meta)) - def bump_snapshot(self): + def bump_snapshot(self) -> None: """ Bumps version and expiration date of TUF 'snapshot' role metadata. diff --git a/warehouse/tuf/tasks.py b/warehouse/tuf/tasks.py index 9fa44fe0a3c6..91a33cdfffb9 100644 --- a/warehouse/tuf/tasks.py +++ b/warehouse/tuf/tasks.py @@ -13,7 +13,7 @@ from warehouse.tasks import task from warehouse.tuf.constants import TUF_REPO_LOCK -from warehouse.tuf.services import IRepositoryService +from warehouse.tuf.services import IRepositoryService, TargetFile # noqa F401 @task(bind=True, ignore_result=True, acks_late=True)