diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index 12bb08f0c..a55242e33 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -898,7 +898,7 @@ def generate_signed_post_policy( ) str_to_sign = base64.b64encode(policy.encode("utf-8")) - signature_bytes = self._credentials.sign_bytes(str_to_sign.encode("ascii")) + signature_bytes = self._credentials.sign_bytes(str_to_sign) signature = binascii.hexlify(signature_bytes).decode("ascii") timestamp, datestamp = get_v4_dtstamps() diff --git a/tests/unit/test__signing.py b/tests/unit/test__signing.py index c3b911f1d..e0001a06d 100644 --- a/tests/unit/test__signing.py +++ b/tests/unit/test__signing.py @@ -762,6 +762,22 @@ def test_bytes(self): self.assertEqual(encoded_param, "bytes") +class TestV4Stamps(unittest.TestCase): + def test_get_v4_dtstamps(self): + import datetime + from google.cloud.storage._signing import get_v4_dtstamps + + with mock.patch( + "google.cloud.storage._signing.NOW", + return_value=datetime.datetime(2020, 3, 12, 13, 14, 15), + ) as now_mock: + timestamp, datestamp = get_v4_dtstamps() + now_mock.assert_called_once() + + self.assertEqual(timestamp, "20200312T131415Z") + self.assertEqual(datestamp, "20200312") + + _DUMMY_SERVICE_ACCOUNT = None diff --git a/tests/unit/test_acl.py b/tests/unit/test_acl.py deleted file mode 100644 index 47400f1ef..000000000 --- a/tests/unit/test_acl.py +++ /dev/null @@ -1,926 +0,0 @@ -# Copyright 2014 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - - -class Test_ACLEntity(unittest.TestCase): - @staticmethod - def _get_target_class(): - from google.cloud.storage.acl import _ACLEntity - - return _ACLEntity - - def _make_one(self, *args, **kw): - return self._get_target_class()(*args, **kw) - - def test_ctor_default_identifier(self): - TYPE = "type" - entity = self._make_one(TYPE) - self.assertEqual(entity.type, TYPE) - self.assertIsNone(entity.identifier) - self.assertEqual(entity.get_roles(), set()) - - def test_ctor_w_identifier(self): - TYPE = "type" - ID = "id" - entity = self._make_one(TYPE, ID) - self.assertEqual(entity.type, TYPE) - self.assertEqual(entity.identifier, ID) - self.assertEqual(entity.get_roles(), set()) - - def test___str__no_identifier(self): - TYPE = "type" - entity = self._make_one(TYPE) - self.assertEqual(str(entity), TYPE) - - def test___str__w_identifier(self): - TYPE = "type" - ID = "id" - entity = self._make_one(TYPE, ID) - self.assertEqual(str(entity), "%s-%s" % (TYPE, ID)) - - def test_grant_simple(self): - TYPE = "type" - ROLE = "role" - entity = self._make_one(TYPE) - entity.grant(ROLE) - self.assertEqual(entity.get_roles(), set([ROLE])) - - def test_grant_duplicate(self): - TYPE = "type" - ROLE1 = "role1" - ROLE2 = "role2" - entity = self._make_one(TYPE) - entity.grant(ROLE1) - entity.grant(ROLE2) - entity.grant(ROLE1) - self.assertEqual(entity.get_roles(), set([ROLE1, ROLE2])) - - def test_revoke_miss(self): - TYPE = "type" - ROLE = "nonesuch" - entity = self._make_one(TYPE) - entity.revoke(ROLE) - self.assertEqual(entity.get_roles(), set()) - - def test_revoke_hit(self): - TYPE = "type" - ROLE1 = "role1" - ROLE2 = "role2" - entity = self._make_one(TYPE) - entity.grant(ROLE1) - entity.grant(ROLE2) - entity.revoke(ROLE1) - self.assertEqual(entity.get_roles(), set([ROLE2])) - - def test_grant_read(self): - TYPE = "type" - entity = self._make_one(TYPE) - entity.grant_read() - self.assertEqual(entity.get_roles(), set([entity.READER_ROLE])) - - def test_grant_write(self): - TYPE = "type" - entity = self._make_one(TYPE) - entity.grant_write() - self.assertEqual(entity.get_roles(), set([entity.WRITER_ROLE])) - - def test_grant_owner(self): - TYPE = "type" - entity = self._make_one(TYPE) - entity.grant_owner() - self.assertEqual(entity.get_roles(), set([entity.OWNER_ROLE])) - - def test_revoke_read(self): - TYPE = "type" - entity = self._make_one(TYPE) - entity.grant(entity.READER_ROLE) - entity.revoke_read() - self.assertEqual(entity.get_roles(), set()) - - def test_revoke_write(self): - TYPE = "type" - entity = self._make_one(TYPE) - entity.grant(entity.WRITER_ROLE) - entity.revoke_write() - self.assertEqual(entity.get_roles(), set()) - - def test_revoke_owner(self): - TYPE = "type" - entity = self._make_one(TYPE) - entity.grant(entity.OWNER_ROLE) - entity.revoke_owner() - self.assertEqual(entity.get_roles(), set()) - - -class FakeReload(object): - """A callable used for faking the reload() method of an ACL instance.""" - - def __init__(self, acl): - self.acl = acl - self.timeouts_used = [] - - def __call__(self, timeout=None): - self.acl.loaded = True - self.timeouts_used.append(timeout) - - -class Test_ACL(unittest.TestCase): - @staticmethod - def _get_target_class(): - from google.cloud.storage.acl import ACL - - return ACL - - @staticmethod - def _get_default_timeout(): - from google.cloud.storage.constants import _DEFAULT_TIMEOUT - - return _DEFAULT_TIMEOUT - - def _make_one(self, *args, **kw): - return self._get_target_class()(*args, **kw) - - def test_validate_predefined(self): - ACL = self._get_target_class() - self.assertIsNone(ACL.validate_predefined(None)) - self.assertEqual(ACL.validate_predefined("public-read"), "publicRead") - self.assertEqual(ACL.validate_predefined("publicRead"), "publicRead") - with self.assertRaises(ValueError): - ACL.validate_predefined("publicread") - - def test_ctor(self): - acl = self._make_one() - self.assertEqual(acl.entities, {}) - self.assertFalse(acl.loaded) - - def test__ensure_loaded(self): - acl = self._make_one() - _reload = FakeReload(acl) - acl.reload = _reload - acl.loaded = False - - acl._ensure_loaded(timeout=42) - - self.assertTrue(acl.loaded) - self.assertEqual(_reload.timeouts_used[0], 42) - - def test_client_is_abstract(self): - acl = self._make_one() - self.assertRaises(NotImplementedError, lambda: acl.client) - - def test_reset(self): - TYPE = "type" - ID = "id" - acl = self._make_one() - acl.loaded = True - acl.entity(TYPE, ID) - acl.reset() - self.assertEqual(acl.entities, {}) - self.assertFalse(acl.loaded) - - def test___iter___empty_eager(self): - acl = self._make_one() - acl.loaded = True - self.assertEqual(list(acl), []) - - def test___iter___empty_lazy(self): - acl = self._make_one() - _reload = FakeReload(acl) - acl.loaded = False - - acl.reload = _reload - self.assertEqual(list(acl), []) - self.assertTrue(acl.loaded) - self.assertEqual(_reload.timeouts_used[0], self._get_default_timeout()) - - def test___iter___non_empty_no_roles(self): - TYPE = "type" - ID = "id" - acl = self._make_one() - acl.loaded = True - acl.entity(TYPE, ID) - self.assertEqual(list(acl), []) - - def test___iter___non_empty_w_roles(self): - TYPE = "type" - ID = "id" - ROLE = "role" - acl = self._make_one() - acl.loaded = True - entity = acl.entity(TYPE, ID) - entity.grant(ROLE) - self.assertEqual(list(acl), [{"entity": "%s-%s" % (TYPE, ID), "role": ROLE}]) - - def test___iter___non_empty_w_empty_role(self): - TYPE = "type" - ID = "id" - acl = self._make_one() - acl.loaded = True - entity = acl.entity(TYPE, ID) - entity.grant("") - self.assertEqual(list(acl), []) - - def test_entity_from_dict_allUsers_eager(self): - ROLE = "role" - acl = self._make_one() - acl.loaded = True - entity = acl.entity_from_dict({"entity": "allUsers", "role": ROLE}) - self.assertEqual(entity.type, "allUsers") - self.assertIsNone(entity.identifier) - self.assertEqual(entity.get_roles(), set([ROLE])) - self.assertEqual(list(acl), [{"entity": "allUsers", "role": ROLE}]) - self.assertEqual(list(acl.get_entities()), [entity]) - - def test_entity_from_dict_allAuthenticatedUsers(self): - ROLE = "role" - acl = self._make_one() - acl.loaded = True - entity = acl.entity_from_dict({"entity": "allAuthenticatedUsers", "role": ROLE}) - self.assertEqual(entity.type, "allAuthenticatedUsers") - self.assertIsNone(entity.identifier) - self.assertEqual(entity.get_roles(), set([ROLE])) - self.assertEqual(list(acl), [{"entity": "allAuthenticatedUsers", "role": ROLE}]) - self.assertEqual(list(acl.get_entities()), [entity]) - - def test_entity_from_dict_string_w_hyphen(self): - ROLE = "role" - acl = self._make_one() - acl.loaded = True - entity = acl.entity_from_dict({"entity": "type-id", "role": ROLE}) - self.assertEqual(entity.type, "type") - self.assertEqual(entity.identifier, "id") - self.assertEqual(entity.get_roles(), set([ROLE])) - self.assertEqual(list(acl), [{"entity": "type-id", "role": ROLE}]) - self.assertEqual(list(acl.get_entities()), [entity]) - - def test_entity_from_dict_string_wo_hyphen(self): - ROLE = "role" - acl = self._make_one() - acl.loaded = True - self.assertRaises( - ValueError, acl.entity_from_dict, {"entity": "bogus", "role": ROLE} - ) - self.assertEqual(list(acl.get_entities()), []) - - def test_has_entity_miss_str_eager(self): - acl = self._make_one() - acl.loaded = True - self.assertFalse(acl.has_entity("nonesuch")) - - def test_has_entity_miss_str_lazy(self): - acl = self._make_one() - _reload = FakeReload(acl) - acl.reload = _reload - acl.loaded = False - - self.assertFalse(acl.has_entity("nonesuch")) - self.assertTrue(acl.loaded) - self.assertEqual(_reload.timeouts_used[0], self._get_default_timeout()) - - def test_has_entity_miss_entity(self): - from google.cloud.storage.acl import _ACLEntity - - TYPE = "type" - ID = "id" - entity = _ACLEntity(TYPE, ID) - acl = self._make_one() - acl.loaded = True - self.assertFalse(acl.has_entity(entity)) - - def test_has_entity_hit_str(self): - TYPE = "type" - ID = "id" - acl = self._make_one() - acl.loaded = True - acl.entity(TYPE, ID) - self.assertTrue(acl.has_entity("%s-%s" % (TYPE, ID))) - - def test_has_entity_hit_entity(self): - TYPE = "type" - ID = "id" - acl = self._make_one() - acl.loaded = True - entity = acl.entity(TYPE, ID) - self.assertTrue(acl.has_entity(entity)) - - def test_get_entity_miss_str_no_default_eager(self): - acl = self._make_one() - acl.loaded = True - self.assertIsNone(acl.get_entity("nonesuch")) - - def test_get_entity_miss_str_no_default_lazy(self): - acl = self._make_one() - _reload = FakeReload(acl) - acl.reload = _reload - acl.loaded = False - - self.assertIsNone(acl.get_entity("nonesuch")) - self.assertTrue(acl.loaded) - self.assertEqual(_reload.timeouts_used[0], self._get_default_timeout()) - - def test_get_entity_miss_entity_no_default(self): - from google.cloud.storage.acl import _ACLEntity - - TYPE = "type" - ID = "id" - entity = _ACLEntity(TYPE, ID) - acl = self._make_one() - acl.loaded = True - self.assertIsNone(acl.get_entity(entity)) - - def test_get_entity_miss_str_w_default(self): - DEFAULT = object() - acl = self._make_one() - acl.loaded = True - self.assertIs(acl.get_entity("nonesuch", DEFAULT), DEFAULT) - - def test_get_entity_miss_entity_w_default(self): - from google.cloud.storage.acl import _ACLEntity - - DEFAULT = object() - TYPE = "type" - ID = "id" - entity = _ACLEntity(TYPE, ID) - acl = self._make_one() - acl.loaded = True - self.assertIs(acl.get_entity(entity, DEFAULT), DEFAULT) - - def test_get_entity_hit_str(self): - TYPE = "type" - ID = "id" - acl = self._make_one() - acl.loaded = True - acl.entity(TYPE, ID) - self.assertTrue(acl.has_entity("%s-%s" % (TYPE, ID))) - - def test_get_entity_hit_entity(self): - TYPE = "type" - ID = "id" - acl = self._make_one() - acl.loaded = True - entity = acl.entity(TYPE, ID) - self.assertTrue(acl.has_entity(entity)) - - def test_add_entity_miss_eager(self): - from google.cloud.storage.acl import _ACLEntity - - TYPE = "type" - ID = "id" - ROLE = "role" - entity = _ACLEntity(TYPE, ID) - entity.grant(ROLE) - acl = self._make_one() - acl.loaded = True - acl.add_entity(entity) - self.assertTrue(acl.loaded) - self.assertEqual(list(acl), [{"entity": "type-id", "role": ROLE}]) - self.assertEqual(list(acl.get_entities()), [entity]) - - def test_add_entity_miss_lazy(self): - from google.cloud.storage.acl import _ACLEntity - - TYPE = "type" - ID = "id" - ROLE = "role" - entity = _ACLEntity(TYPE, ID) - entity.grant(ROLE) - acl = self._make_one() - - _reload = FakeReload(acl) - acl.reload = _reload - acl.loaded = False - - acl.add_entity(entity) - self.assertTrue(acl.loaded) - self.assertEqual(list(acl), [{"entity": "type-id", "role": ROLE}]) - self.assertEqual(list(acl.get_entities()), [entity]) - self.assertTrue(acl.loaded) - self.assertEqual(_reload.timeouts_used[0], self._get_default_timeout()) - - def test_add_entity_hit(self): - from google.cloud.storage.acl import _ACLEntity - - TYPE = "type" - ID = "id" - ENTITY_VAL = "%s-%s" % (TYPE, ID) - ROLE = "role" - entity = _ACLEntity(TYPE, ID) - entity.grant(ROLE) - acl = self._make_one() - acl.loaded = True - before = acl.entity(TYPE, ID) - acl.add_entity(entity) - self.assertTrue(acl.loaded) - self.assertIsNot(acl.get_entity(ENTITY_VAL), before) - self.assertIs(acl.get_entity(ENTITY_VAL), entity) - self.assertEqual(list(acl), [{"entity": "type-id", "role": ROLE}]) - self.assertEqual(list(acl.get_entities()), [entity]) - - def test_entity_miss(self): - TYPE = "type" - ID = "id" - ROLE = "role" - acl = self._make_one() - acl.loaded = True - entity = acl.entity(TYPE, ID) - self.assertTrue(acl.loaded) - entity.grant(ROLE) - self.assertEqual(list(acl), [{"entity": "type-id", "role": ROLE}]) - self.assertEqual(list(acl.get_entities()), [entity]) - - def test_entity_hit(self): - TYPE = "type" - ID = "id" - ROLE = "role" - acl = self._make_one() - acl.loaded = True - before = acl.entity(TYPE, ID) - before.grant(ROLE) - entity = acl.entity(TYPE, ID) - self.assertIs(entity, before) - self.assertEqual(list(acl), [{"entity": "type-id", "role": ROLE}]) - self.assertEqual(list(acl.get_entities()), [entity]) - - def test_user(self): - ID = "id" - ROLE = "role" - acl = self._make_one() - acl.loaded = True - entity = acl.user(ID) - entity.grant(ROLE) - self.assertEqual(entity.type, "user") - self.assertEqual(entity.identifier, ID) - self.assertEqual(list(acl), [{"entity": "user-%s" % ID, "role": ROLE}]) - - def test_group(self): - ID = "id" - ROLE = "role" - acl = self._make_one() - acl.loaded = True - entity = acl.group(ID) - entity.grant(ROLE) - self.assertEqual(entity.type, "group") - self.assertEqual(entity.identifier, ID) - self.assertEqual(list(acl), [{"entity": "group-%s" % ID, "role": ROLE}]) - - def test_domain(self): - ID = "id" - ROLE = "role" - acl = self._make_one() - acl.loaded = True - entity = acl.domain(ID) - entity.grant(ROLE) - self.assertEqual(entity.type, "domain") - self.assertEqual(entity.identifier, ID) - self.assertEqual(list(acl), [{"entity": "domain-%s" % ID, "role": ROLE}]) - - def test_all(self): - ROLE = "role" - acl = self._make_one() - acl.loaded = True - entity = acl.all() - entity.grant(ROLE) - self.assertEqual(entity.type, "allUsers") - self.assertIsNone(entity.identifier) - self.assertEqual(list(acl), [{"entity": "allUsers", "role": ROLE}]) - - def test_all_authenticated(self): - ROLE = "role" - acl = self._make_one() - acl.loaded = True - entity = acl.all_authenticated() - entity.grant(ROLE) - self.assertEqual(entity.type, "allAuthenticatedUsers") - self.assertIsNone(entity.identifier) - self.assertEqual(list(acl), [{"entity": "allAuthenticatedUsers", "role": ROLE}]) - - def test_get_entities_empty_eager(self): - acl = self._make_one() - acl.loaded = True - self.assertEqual(acl.get_entities(), []) - - def test_get_entities_empty_lazy(self): - acl = self._make_one() - _reload = FakeReload(acl) - acl.reload = _reload - acl.loaded = False - - self.assertEqual(acl.get_entities(), []) - self.assertTrue(acl.loaded) - self.assertEqual(_reload.timeouts_used[0], self._get_default_timeout()) - - def test_get_entities_nonempty(self): - TYPE = "type" - ID = "id" - acl = self._make_one() - acl.loaded = True - entity = acl.entity(TYPE, ID) - self.assertEqual(acl.get_entities(), [entity]) - - def test_reload_missing(self): - # https://github.com/GoogleCloudPlatform/google-cloud-python/issues/652 - ROLE = "role" - connection = _Connection({}) - client = _Client(connection) - acl = self._make_one() - acl.reload_path = "/testing/acl" - acl.loaded = True - acl.entity("allUsers", ROLE) - acl.reload(client=client, timeout=42) - self.assertEqual(list(acl), []) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "GET", - "path": "/testing/acl", - "query_params": {}, - "timeout": 42, - }, - ) - - def test_reload_empty_result_clears_local(self): - ROLE = "role" - connection = _Connection({"items": []}) - client = _Client(connection) - acl = self._make_one() - acl.reload_path = "/testing/acl" - acl.loaded = True - acl.entity("allUsers", ROLE) - - acl.reload(client=client) - - self.assertTrue(acl.loaded) - self.assertEqual(list(acl), []) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "GET", - "path": "/testing/acl", - "query_params": {}, - "timeout": self._get_default_timeout(), - }, - ) - - def test_reload_nonempty_result_w_user_project(self): - ROLE = "role" - USER_PROJECT = "user-project-123" - connection = _Connection({"items": [{"entity": "allUsers", "role": ROLE}]}) - client = _Client(connection) - acl = self._make_one() - acl.reload_path = "/testing/acl" - acl.loaded = True - acl.user_project = USER_PROJECT - - acl.reload(client=client) - - self.assertTrue(acl.loaded) - self.assertEqual(list(acl), [{"entity": "allUsers", "role": ROLE}]) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "GET", - "path": "/testing/acl", - "query_params": {"userProject": USER_PROJECT}, - "timeout": self._get_default_timeout(), - }, - ) - - def test_save_none_set_none_passed(self): - connection = _Connection() - client = _Client(connection) - acl = self._make_one() - acl.save_path = "/testing" - acl.save(client=client) - kw = connection._requested - self.assertEqual(len(kw), 0) - - def test_save_existing_missing_none_passed(self): - connection = _Connection({}) - client = _Client(connection) - acl = self._make_one() - acl.save_path = "/testing" - acl.loaded = True - acl.save(client=client, timeout=42) - self.assertEqual(list(acl), []) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "PATCH") - self.assertEqual(kw[0]["path"], "/testing") - self.assertEqual(kw[0]["data"], {"acl": []}) - self.assertEqual(kw[0]["query_params"], {"projection": "full"}) - self.assertEqual(kw[0]["timeout"], 42) - - def test_save_no_acl(self): - ROLE = "role" - AFTER = [{"entity": "allUsers", "role": ROLE}] - connection = _Connection({"acl": AFTER}) - client = _Client(connection) - acl = self._make_one() - acl.save_path = "/testing" - acl.loaded = True - acl.entity("allUsers").grant(ROLE) - acl.save(client=client) - self.assertEqual(list(acl), AFTER) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "PATCH") - self.assertEqual(kw[0]["path"], "/testing") - self.assertEqual( - kw[0], - { - "method": "PATCH", - "path": "/testing", - "query_params": {"projection": "full"}, - "data": {"acl": AFTER}, - "timeout": self._get_default_timeout(), - }, - ) - - def test_save_w_acl_w_user_project(self): - ROLE1 = "role1" - ROLE2 = "role2" - STICKY = {"entity": "allUsers", "role": ROLE2} - USER_PROJECT = "user-project-123" - new_acl = [{"entity": "allUsers", "role": ROLE1}] - connection = _Connection({"acl": [STICKY] + new_acl}) - client = _Client(connection) - acl = self._make_one() - acl.save_path = "/testing" - acl.loaded = True - acl.user_project = USER_PROJECT - - acl.save(new_acl, client=client) - - entries = list(acl) - self.assertEqual(len(entries), 2) - self.assertTrue(STICKY in entries) - self.assertTrue(new_acl[0] in entries) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "PATCH", - "path": "/testing", - "query_params": {"projection": "full", "userProject": USER_PROJECT}, - "data": {"acl": new_acl}, - "timeout": self._get_default_timeout(), - }, - ) - - def test_save_prefefined_invalid(self): - connection = _Connection() - client = _Client(connection) - acl = self._make_one() - acl.save_path = "/testing" - acl.loaded = True - with self.assertRaises(ValueError): - acl.save_predefined("bogus", client=client) - - def test_save_predefined_valid(self): - PREDEFINED = "private" - connection = _Connection({"acl": []}) - client = _Client(connection) - acl = self._make_one() - acl.save_path = "/testing" - acl.loaded = True - acl.save_predefined(PREDEFINED, client=client, timeout=42) - entries = list(acl) - self.assertEqual(len(entries), 0) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "PATCH", - "path": "/testing", - "query_params": {"projection": "full", "predefinedAcl": PREDEFINED}, - "data": {"acl": []}, - "timeout": 42, - }, - ) - - def test_save_predefined_w_XML_alias(self): - PREDEFINED_XML = "project-private" - PREDEFINED_JSON = "projectPrivate" - connection = _Connection({"acl": []}) - client = _Client(connection) - acl = self._make_one() - acl.save_path = "/testing" - acl.loaded = True - acl.save_predefined(PREDEFINED_XML, client=client) - entries = list(acl) - self.assertEqual(len(entries), 0) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "PATCH", - "path": "/testing", - "query_params": { - "projection": "full", - "predefinedAcl": PREDEFINED_JSON, - }, - "data": {"acl": []}, - "timeout": self._get_default_timeout(), - }, - ) - - def test_save_predefined_valid_w_alternate_query_param(self): - # Cover case where subclass overrides _PREDEFINED_QUERY_PARAM - PREDEFINED = "publicRead" - connection = _Connection({"acl": []}) - client = _Client(connection) - acl = self._make_one() - acl.save_path = "/testing" - acl.loaded = True - acl._PREDEFINED_QUERY_PARAM = "alternate" - acl.save_predefined(PREDEFINED, client=client) - entries = list(acl) - self.assertEqual(len(entries), 0) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "PATCH", - "path": "/testing", - "query_params": {"projection": "full", "alternate": PREDEFINED}, - "data": {"acl": []}, - "timeout": self._get_default_timeout(), - }, - ) - - def test_clear(self): - ROLE1 = "role1" - ROLE2 = "role2" - STICKY = {"entity": "allUsers", "role": ROLE2} - connection = _Connection({"acl": [STICKY]}) - client = _Client(connection) - acl = self._make_one() - acl.save_path = "/testing" - acl.loaded = True - acl.entity("allUsers", ROLE1) - acl.clear(client=client, timeout=42) - self.assertEqual(list(acl), [STICKY]) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "PATCH", - "path": "/testing", - "query_params": {"projection": "full"}, - "data": {"acl": []}, - "timeout": 42, - }, - ) - - -class Test_BucketACL(unittest.TestCase): - @staticmethod - def _get_target_class(): - from google.cloud.storage.acl import BucketACL - - return BucketACL - - def _make_one(self, *args, **kw): - return self._get_target_class()(*args, **kw) - - def test_ctor(self): - NAME = "name" - bucket = _Bucket(NAME) - acl = self._make_one(bucket) - self.assertEqual(acl.entities, {}) - self.assertFalse(acl.loaded) - self.assertIs(acl.bucket, bucket) - self.assertEqual(acl.reload_path, "/b/%s/acl" % NAME) - self.assertEqual(acl.save_path, "/b/%s" % NAME) - - def test_user_project(self): - NAME = "name" - USER_PROJECT = "user-project-123" - bucket = _Bucket(NAME) - acl = self._make_one(bucket) - self.assertIsNone(acl.user_project) - bucket.user_project = USER_PROJECT - self.assertEqual(acl.user_project, USER_PROJECT) - - -class Test_DefaultObjectACL(unittest.TestCase): - @staticmethod - def _get_target_class(): - from google.cloud.storage.acl import DefaultObjectACL - - return DefaultObjectACL - - def _make_one(self, *args, **kw): - return self._get_target_class()(*args, **kw) - - def test_ctor(self): - NAME = "name" - bucket = _Bucket(NAME) - acl = self._make_one(bucket) - self.assertEqual(acl.entities, {}) - self.assertFalse(acl.loaded) - self.assertIs(acl.bucket, bucket) - self.assertEqual(acl.reload_path, "/b/%s/defaultObjectAcl" % NAME) - self.assertEqual(acl.save_path, "/b/%s" % NAME) - - -class Test_ObjectACL(unittest.TestCase): - @staticmethod - def _get_target_class(): - from google.cloud.storage.acl import ObjectACL - - return ObjectACL - - def _make_one(self, *args, **kw): - return self._get_target_class()(*args, **kw) - - def test_ctor(self): - NAME = "name" - BLOB_NAME = "blob-name" - bucket = _Bucket(NAME) - blob = _Blob(bucket, BLOB_NAME) - acl = self._make_one(blob) - self.assertEqual(acl.entities, {}) - self.assertFalse(acl.loaded) - self.assertIs(acl.blob, blob) - self.assertEqual(acl.reload_path, "/b/%s/o/%s/acl" % (NAME, BLOB_NAME)) - self.assertEqual(acl.save_path, "/b/%s/o/%s" % (NAME, BLOB_NAME)) - - def test_user_project(self): - NAME = "name" - BLOB_NAME = "blob-name" - USER_PROJECT = "user-project-123" - bucket = _Bucket(NAME) - blob = _Blob(bucket, BLOB_NAME) - acl = self._make_one(blob) - self.assertIsNone(acl.user_project) - blob.user_project = USER_PROJECT - self.assertEqual(acl.user_project, USER_PROJECT) - - -class _Blob(object): - - user_project = None - - def __init__(self, bucket, blob): - self.bucket = bucket - self.blob = blob - - @property - def path(self): - return "%s/o/%s" % (self.bucket.path, self.blob) - - -class _Bucket(object): - - user_project = None - - def __init__(self, name): - self.name = name - - @property - def path(self): - return "/b/%s" % self.name - - -class _Connection(object): - _delete_ok = False - - def __init__(self, *responses): - self._responses = responses - self._requested = [] - self._deleted = [] - - def api_request(self, **kw): - self._requested.append(kw) - response, self._responses = self._responses[0], self._responses[1:] - return response - - -class _Client(object): - def __init__(self, connection): - self._connection = connection diff --git a/tests/unit/test_batch.py b/tests/unit/test_batch.py deleted file mode 100644 index ec8fe75de..000000000 --- a/tests/unit/test_batch.py +++ /dev/null @@ -1,649 +0,0 @@ -# Copyright 2014 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -import mock -import requests -from six.moves import http_client - - -def _make_credentials(): - import google.auth.credentials - - return mock.Mock(spec=google.auth.credentials.Credentials) - - -def _make_response(status=http_client.OK, content=b"", headers={}): - response = requests.Response() - response.status_code = status - response._content = content - response.headers = headers - response.request = requests.Request() - return response - - -def _make_requests_session(responses): - session = mock.create_autospec(requests.Session, instance=True) - session.request.side_effect = responses - return session - - -class TestMIMEApplicationHTTP(unittest.TestCase): - @staticmethod - def _get_target_class(): - from google.cloud.storage.batch import MIMEApplicationHTTP - - return MIMEApplicationHTTP - - def _make_one(self, *args, **kw): - return self._get_target_class()(*args, **kw) - - def test_ctor_body_None(self): - METHOD = "DELETE" - PATH = "/path/to/api" - LINES = ["DELETE /path/to/api HTTP/1.1", ""] - mah = self._make_one(METHOD, PATH, {}, None) - self.assertEqual(mah.get_content_type(), "application/http") - self.assertEqual(mah.get_payload().splitlines(), LINES) - - def test_ctor_body_str(self): - METHOD = "GET" - PATH = "/path/to/api" - BODY = "ABC" - HEADERS = {"Content-Length": len(BODY), "Content-Type": "text/plain"} - LINES = [ - "GET /path/to/api HTTP/1.1", - "Content-Length: 3", - "Content-Type: text/plain", - "", - "ABC", - ] - mah = self._make_one(METHOD, PATH, HEADERS, BODY) - self.assertEqual(mah.get_payload().splitlines(), LINES) - - def test_ctor_body_dict(self): - METHOD = "GET" - PATH = "/path/to/api" - BODY = {"foo": "bar"} - HEADERS = {} - LINES = [ - "GET /path/to/api HTTP/1.1", - "Content-Length: 14", - "Content-Type: application/json", - "", - '{"foo": "bar"}', - ] - mah = self._make_one(METHOD, PATH, HEADERS, BODY) - self.assertEqual(mah.get_payload().splitlines(), LINES) - - -class TestBatch(unittest.TestCase): - @staticmethod - def _get_default_timeout(): - from google.cloud.storage.constants import _DEFAULT_TIMEOUT - - return _DEFAULT_TIMEOUT - - @staticmethod - def _get_target_class(): - from google.cloud.storage.batch import Batch - - return Batch - - def _make_one(self, *args, **kw): - return self._get_target_class()(*args, **kw) - - def test_ctor(self): - http = _make_requests_session([]) - connection = _Connection(http=http) - client = _Client(connection) - batch = self._make_one(client) - self.assertIs(batch._client, client) - self.assertEqual(len(batch._requests), 0) - self.assertEqual(len(batch._target_objects), 0) - - def test_current(self): - from google.cloud.storage.client import Client - - project = "PROJECT" - credentials = _make_credentials() - client = Client(project=project, credentials=credentials) - batch1 = self._make_one(client) - self.assertIsNone(batch1.current()) - - client._push_batch(batch1) - self.assertIs(batch1.current(), batch1) - - batch2 = self._make_one(client) - client._push_batch(batch2) - self.assertIs(batch1.current(), batch2) - - def test__make_request_GET_normal(self): - from google.cloud.storage.batch import _FutureDict - - url = "http://example.com/api" - http = _make_requests_session([]) - connection = _Connection(http=http) - batch = self._make_one(connection) - target = _MockObject() - - response = batch._make_request("GET", url, target_object=target) - - # Check the respone - self.assertEqual(response.status_code, 204) - self.assertIsInstance(response.json(), _FutureDict) - self.assertIsInstance(response.content, _FutureDict) - self.assertIs(target._properties, response.content) - - # The real http request should not have been called yet. - http.request.assert_not_called() - - # Check the queued request - self.assertEqual(len(batch._requests), 1) - request = batch._requests[0] - request_method, request_url, _, request_data, _ = request - self.assertEqual(request_method, "GET") - self.assertEqual(request_url, url) - self.assertIsNone(request_data) - - def test__make_request_POST_normal(self): - from google.cloud.storage.batch import _FutureDict - - url = "http://example.com/api" - http = _make_requests_session([]) - connection = _Connection(http=http) - batch = self._make_one(connection) - data = {"foo": 1} - target = _MockObject() - - response = batch._make_request( - "POST", url, data={"foo": 1}, target_object=target - ) - - self.assertEqual(response.status_code, 204) - self.assertIsInstance(response.content, _FutureDict) - self.assertIs(target._properties, response.content) - - # The real http request should not have been called yet. - http.request.assert_not_called() - - request = batch._requests[0] - request_method, request_url, _, request_data, _ = request - self.assertEqual(request_method, "POST") - self.assertEqual(request_url, url) - self.assertEqual(request_data, data) - - def test__make_request_PATCH_normal(self): - from google.cloud.storage.batch import _FutureDict - - url = "http://example.com/api" - http = _make_requests_session([]) - connection = _Connection(http=http) - batch = self._make_one(connection) - data = {"foo": 1} - target = _MockObject() - - response = batch._make_request( - "PATCH", url, data={"foo": 1}, target_object=target - ) - - self.assertEqual(response.status_code, 204) - self.assertIsInstance(response.content, _FutureDict) - self.assertIs(target._properties, response.content) - - # The real http request should not have been called yet. - http.request.assert_not_called() - - request = batch._requests[0] - request_method, request_url, _, request_data, _ = request - self.assertEqual(request_method, "PATCH") - self.assertEqual(request_url, url) - self.assertEqual(request_data, data) - - def test__make_request_DELETE_normal(self): - from google.cloud.storage.batch import _FutureDict - - url = "http://example.com/api" - http = _make_requests_session([]) - connection = _Connection(http=http) - batch = self._make_one(connection) - target = _MockObject() - - response = batch._make_request("DELETE", url, target_object=target) - - # Check the respone - self.assertEqual(response.status_code, 204) - self.assertIsInstance(response.content, _FutureDict) - self.assertIs(target._properties, response.content) - - # The real http request should not have been called yet. - http.request.assert_not_called() - - # Check the queued request - self.assertEqual(len(batch._requests), 1) - request = batch._requests[0] - request_method, request_url, _, request_data, _ = request - self.assertEqual(request_method, "DELETE") - self.assertEqual(request_url, url) - self.assertIsNone(request_data) - - def test__make_request_POST_too_many_requests(self): - url = "http://example.com/api" - http = _make_requests_session([]) - connection = _Connection(http=http) - batch = self._make_one(connection) - - batch._MAX_BATCH_SIZE = 1 - batch._requests.append(("POST", url, {}, {"bar": 2})) - - with self.assertRaises(ValueError): - batch._make_request("POST", url, data={"foo": 1}) - - def test_finish_empty(self): - http = _make_requests_session([]) - connection = _Connection(http=http) - batch = self._make_one(connection) - - with self.assertRaises(ValueError): - batch.finish() - - def _get_payload_chunks(self, boundary, payload): - divider = "--" + boundary[len('boundary="') : -1] - chunks = payload.split(divider)[1:-1] # discard prolog / epilog - return chunks - - def _check_subrequest_no_payload(self, chunk, method, url): - lines = chunk.splitlines() - # blank + 2 headers + blank + request + blank + blank - self.assertEqual(len(lines), 7) - self.assertEqual(lines[0], "") - self.assertEqual(lines[1], "Content-Type: application/http") - self.assertEqual(lines[2], "MIME-Version: 1.0") - self.assertEqual(lines[3], "") - self.assertEqual(lines[4], "%s %s HTTP/1.1" % (method, url)) - self.assertEqual(lines[5], "") - self.assertEqual(lines[6], "") - - def _check_subrequest_payload(self, chunk, method, url, payload): - import json - - lines = chunk.splitlines() - # blank + 2 headers + blank + request + 2 headers + blank + body - payload_str = json.dumps(payload) - self.assertEqual(lines[0], "") - self.assertEqual(lines[1], "Content-Type: application/http") - self.assertEqual(lines[2], "MIME-Version: 1.0") - self.assertEqual(lines[3], "") - self.assertEqual(lines[4], "%s %s HTTP/1.1" % (method, url)) - if method == "GET": - self.assertEqual(len(lines), 7) - self.assertEqual(lines[5], "") - self.assertEqual(lines[6], "") - else: - self.assertEqual(len(lines), 9) - self.assertEqual(lines[5], "Content-Length: %d" % len(payload_str)) - self.assertEqual(lines[6], "Content-Type: application/json") - self.assertEqual(lines[7], "") - self.assertEqual(json.loads(lines[8]), payload) - - def _get_mutlipart_request(self, http): - request_call = http.request.mock_calls[0][2] - request_headers = request_call["headers"] - request_body = request_call["data"] - content_type, boundary = [ - value.strip() for value in request_headers["Content-Type"].split(";") - ] - - return request_headers, request_body, content_type, boundary - - def test_finish_nonempty(self): - url = "http://api.example.com/other_api" - expected_response = _make_response( - content=_THREE_PART_MIME_RESPONSE, - headers={"content-type": 'multipart/mixed; boundary="DEADBEEF="'}, - ) - http = _make_requests_session([expected_response]) - connection = _Connection(http=http) - client = _Client(connection) - batch = self._make_one(client) - batch.API_BASE_URL = "http://api.example.com" - - batch._do_request("POST", url, {}, {"foo": 1, "bar": 2}, None) - batch._do_request("PATCH", url, {}, {"bar": 3}, None) - batch._do_request("DELETE", url, {}, None, None) - result = batch.finish() - - self.assertEqual(len(result), len(batch._requests)) - - response1, response2, response3 = result - - self.assertEqual( - response1.headers, - {"Content-Length": "20", "Content-Type": "application/json; charset=UTF-8"}, - ) - self.assertEqual(response1.json(), {"foo": 1, "bar": 2}) - - self.assertEqual( - response2.headers, - {"Content-Length": "20", "Content-Type": "application/json; charset=UTF-8"}, - ) - self.assertEqual(response2.json(), {"foo": 1, "bar": 3}) - - self.assertEqual(response3.headers, {"Content-Length": "0"}) - self.assertEqual(response3.status_code, http_client.NO_CONTENT) - - expected_url = "{}/batch/storage/v1".format(batch.API_BASE_URL) - http.request.assert_called_once_with( - method="POST", - url=expected_url, - headers=mock.ANY, - data=mock.ANY, - timeout=self._get_default_timeout(), - ) - - request_info = self._get_mutlipart_request(http) - request_headers, request_body, content_type, boundary = request_info - - self.assertEqual(content_type, "multipart/mixed") - self.assertTrue(boundary.startswith('boundary="==')) - self.assertTrue(boundary.endswith('=="')) - self.assertEqual(request_headers["MIME-Version"], "1.0") - - chunks = self._get_payload_chunks(boundary, request_body) - self.assertEqual(len(chunks), 3) - self._check_subrequest_payload(chunks[0], "POST", url, {"foo": 1, "bar": 2}) - self._check_subrequest_payload(chunks[1], "PATCH", url, {"bar": 3}) - self._check_subrequest_no_payload(chunks[2], "DELETE", url) - - def test_finish_responses_mismatch(self): - url = "http://api.example.com/other_api" - expected_response = _make_response( - content=_TWO_PART_MIME_RESPONSE_WITH_FAIL, - headers={"content-type": 'multipart/mixed; boundary="DEADBEEF="'}, - ) - http = _make_requests_session([expected_response]) - connection = _Connection(http=http) - client = _Client(connection) - batch = self._make_one(client) - batch.API_BASE_URL = "http://api.example.com" - - batch._requests.append(("GET", url, {}, None)) - with self.assertRaises(ValueError): - batch.finish() - - def test_finish_nonempty_with_status_failure(self): - from google.cloud.exceptions import NotFound - - url = "http://api.example.com/other_api" - expected_response = _make_response( - content=_TWO_PART_MIME_RESPONSE_WITH_FAIL, - headers={"content-type": 'multipart/mixed; boundary="DEADBEEF="'}, - ) - http = _make_requests_session([expected_response]) - connection = _Connection(http=http) - client = _Client(connection) - batch = self._make_one(client) - batch.API_BASE_URL = "http://api.example.com" - target1 = _MockObject() - target2 = _MockObject() - - batch._do_request("GET", url, {}, None, target1, timeout=42) - batch._do_request("GET", url, {}, None, target2, timeout=420) - - # Make sure futures are not populated. - self.assertEqual( - [future for future in batch._target_objects], [target1, target2] - ) - target2_future_before = target2._properties - - with self.assertRaises(NotFound): - batch.finish() - - self.assertEqual(target1._properties, {"foo": 1, "bar": 2}) - self.assertIs(target2._properties, target2_future_before) - - expected_url = "{}/batch/storage/v1".format(batch.API_BASE_URL) - http.request.assert_called_once_with( - method="POST", - url=expected_url, - headers=mock.ANY, - data=mock.ANY, - timeout=420, # the last request timeout prevails - ) - - _, request_body, _, boundary = self._get_mutlipart_request(http) - - chunks = self._get_payload_chunks(boundary, request_body) - self.assertEqual(len(chunks), 2) - self._check_subrequest_payload(chunks[0], "GET", url, {}) - self._check_subrequest_payload(chunks[1], "GET", url, {}) - - def test_finish_nonempty_non_multipart_response(self): - url = "http://api.example.com/other_api" - http = _make_requests_session([_make_response()]) - connection = _Connection(http=http) - client = _Client(connection) - batch = self._make_one(client) - batch._requests.append(("POST", url, {}, {"foo": 1, "bar": 2})) - - with self.assertRaises(ValueError): - batch.finish() - - def test_as_context_mgr_wo_error(self): - from google.cloud.storage.client import Client - - url = "http://example.com/api" - expected_response = _make_response( - content=_THREE_PART_MIME_RESPONSE, - headers={"content-type": 'multipart/mixed; boundary="DEADBEEF="'}, - ) - http = _make_requests_session([expected_response]) - project = "PROJECT" - credentials = _make_credentials() - client = Client(project=project, credentials=credentials) - client._http_internal = http - - self.assertEqual(list(client._batch_stack), []) - - target1 = _MockObject() - target2 = _MockObject() - target3 = _MockObject() - - with self._make_one(client) as batch: - self.assertEqual(list(client._batch_stack), [batch]) - batch._make_request( - "POST", url, {"foo": 1, "bar": 2}, target_object=target1 - ) - batch._make_request("PATCH", url, {"bar": 3}, target_object=target2) - batch._make_request("DELETE", url, target_object=target3) - - self.assertEqual(list(client._batch_stack), []) - self.assertEqual(len(batch._requests), 3) - self.assertEqual(batch._requests[0][0], "POST") - self.assertEqual(batch._requests[1][0], "PATCH") - self.assertEqual(batch._requests[2][0], "DELETE") - self.assertEqual(batch._target_objects, [target1, target2, target3]) - self.assertEqual(target1._properties, {"foo": 1, "bar": 2}) - self.assertEqual(target2._properties, {"foo": 1, "bar": 3}) - self.assertEqual(target3._properties, b"") - - def test_as_context_mgr_w_error(self): - from google.cloud.storage.batch import _FutureDict - from google.cloud.storage.client import Client - - URL = "http://example.com/api" - http = _make_requests_session([]) - connection = _Connection(http=http) - project = "PROJECT" - credentials = _make_credentials() - client = Client(project=project, credentials=credentials) - client._base_connection = connection - - self.assertEqual(list(client._batch_stack), []) - - target1 = _MockObject() - target2 = _MockObject() - target3 = _MockObject() - try: - with self._make_one(client) as batch: - self.assertEqual(list(client._batch_stack), [batch]) - batch._make_request( - "POST", URL, {"foo": 1, "bar": 2}, target_object=target1 - ) - batch._make_request("PATCH", URL, {"bar": 3}, target_object=target2) - batch._make_request("DELETE", URL, target_object=target3) - raise ValueError() - except ValueError: - pass - - http.request.assert_not_called() - self.assertEqual(list(client._batch_stack), []) - self.assertEqual(len(batch._requests), 3) - self.assertEqual(batch._target_objects, [target1, target2, target3]) - # Since the context manager fails, finish will not get called and - # the _properties will still be futures. - self.assertIsInstance(target1._properties, _FutureDict) - self.assertIsInstance(target2._properties, _FutureDict) - self.assertIsInstance(target3._properties, _FutureDict) - - -class Test__unpack_batch_response(unittest.TestCase): - def _call_fut(self, headers, content): - from google.cloud.storage.batch import _unpack_batch_response - - response = _make_response(content=content, headers=headers) - - return _unpack_batch_response(response) - - def _unpack_helper(self, response, content): - result = list(self._call_fut(response, content)) - self.assertEqual(len(result), 3) - - self.assertEqual(result[0].status_code, http_client.OK) - self.assertEqual(result[0].json(), {u"bar": 2, u"foo": 1}) - self.assertEqual(result[1].status_code, http_client.OK) - self.assertEqual(result[1].json(), {u"foo": 1, u"bar": 3}) - self.assertEqual(result[2].status_code, http_client.NO_CONTENT) - - def test_bytes_headers(self): - RESPONSE = {"content-type": b'multipart/mixed; boundary="DEADBEEF="'} - CONTENT = _THREE_PART_MIME_RESPONSE - self._unpack_helper(RESPONSE, CONTENT) - - def test_unicode_headers(self): - RESPONSE = {"content-type": u'multipart/mixed; boundary="DEADBEEF="'} - CONTENT = _THREE_PART_MIME_RESPONSE - self._unpack_helper(RESPONSE, CONTENT) - - -_TWO_PART_MIME_RESPONSE_WITH_FAIL = b"""\ ---DEADBEEF= -Content-Type: application/json -Content-ID: - -HTTP/1.1 200 OK -Content-Type: application/json; charset=UTF-8 -Content-Length: 20 - -{"foo": 1, "bar": 2} - ---DEADBEEF= -Content-Type: application/json -Content-ID: - -HTTP/1.1 404 Not Found -Content-Type: application/json; charset=UTF-8 -Content-Length: 35 - -{"error": {"message": "Not Found"}} - ---DEADBEEF=-- -""" - -_THREE_PART_MIME_RESPONSE = b"""\ ---DEADBEEF= -Content-Type: application/json -Content-ID: - -HTTP/1.1 200 OK -Content-Type: application/json; charset=UTF-8 -Content-Length: 20 - -{"foo": 1, "bar": 2} - ---DEADBEEF= -Content-Type: application/json -Content-ID: - -HTTP/1.1 200 OK -Content-Type: application/json; charset=UTF-8 -Content-Length: 20 - -{"foo": 1, "bar": 3} - ---DEADBEEF= -Content-Type: text/plain -Content-ID: - -HTTP/1.1 204 No Content -Content-Length: 0 - ---DEADBEEF=-- -""" - - -class Test__FutureDict(unittest.TestCase): - def _make_one(self, *args, **kw): - from google.cloud.storage.batch import _FutureDict - - return _FutureDict(*args, **kw) - - def test_get(self): - future = self._make_one() - self.assertRaises(KeyError, future.get, None) - - def test___getitem__(self): - future = self._make_one() - value = orig_value = object() - with self.assertRaises(KeyError): - value = future[None] - self.assertIs(value, orig_value) - - def test___setitem__(self): - future = self._make_one() - with self.assertRaises(KeyError): - future[None] = None - - -class _Connection(object): - - project = "TESTING" - - def __init__(self, **kw): - self.__dict__.update(kw) - - def _make_request(self, method, url, data=None, headers=None, timeout=None): - return self.http.request( - url=url, method=method, headers=headers, data=data, timeout=timeout - ) - - -class _MockObject(object): - pass - - -class _Client(object): - def __init__(self, connection): - self._base_connection = connection diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py deleted file mode 100644 index f656e6441..000000000 --- a/tests/unit/test_blob.py +++ /dev/null @@ -1,3433 +0,0 @@ -# Copyright 2014 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import base64 -import datetime -import hashlib -import io -import json -import os -import tempfile -import unittest - -import mock -import pytest -import six -from six.moves import http_client - - -def _make_credentials(): - import google.auth.credentials - - return mock.Mock(spec=google.auth.credentials.Credentials) - - -class Test_Blob(unittest.TestCase): - @staticmethod - def _make_one(*args, **kw): - from google.cloud.storage.blob import Blob - - properties = kw.pop("properties", {}) - blob = Blob(*args, **kw) - blob._properties.update(properties) - return blob - - @staticmethod - def _get_default_timeout(): - from google.cloud.storage.constants import _DEFAULT_TIMEOUT - - return _DEFAULT_TIMEOUT - - def test_ctor_wo_encryption_key(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - properties = {"key": "value"} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertIs(blob.bucket, bucket) - self.assertEqual(blob.name, BLOB_NAME) - self.assertEqual(blob._properties, properties) - self.assertFalse(blob._acl.loaded) - self.assertIs(blob._acl.blob, blob) - self.assertEqual(blob._encryption_key, None) - self.assertEqual(blob.kms_key_name, None) - - def test_ctor_with_encoded_unicode(self): - blob_name = b"wet \xe2\x9b\xb5" - blob = self._make_one(blob_name, bucket=None) - unicode_name = u"wet \N{sailboat}" - self.assertNotIsInstance(blob.name, bytes) - self.assertIsInstance(blob.name, six.text_type) - self.assertEqual(blob.name, unicode_name) - - def test_ctor_w_encryption_key(self): - KEY = b"01234567890123456789012345678901" # 32 bytes - BLOB_NAME = "blob-name" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=KEY) - self.assertEqual(blob._encryption_key, KEY) - self.assertEqual(blob.kms_key_name, None) - - def test_ctor_w_kms_key_name_and_encryption_key(self): - KEY = b"01234567890123456789012345678901" # 32 bytes - KMS_RESOURCE = ( - "projects/test-project-123/" - "locations/us/" - "keyRings/test-ring/" - "cryptoKeys/test-key" - ) - BLOB_NAME = "blob-name" - bucket = _Bucket() - - with self.assertRaises(ValueError): - self._make_one( - BLOB_NAME, bucket=bucket, encryption_key=KEY, kms_key_name=KMS_RESOURCE - ) - - def test_ctor_w_kms_key_name(self): - KMS_RESOURCE = ( - "projects/test-project-123/" - "locations/us/" - "keyRings/test-ring/" - "cryptoKeys/test-key" - ) - BLOB_NAME = "blob-name" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket, kms_key_name=KMS_RESOURCE) - self.assertEqual(blob._encryption_key, None) - self.assertEqual(blob.kms_key_name, KMS_RESOURCE) - - def test_ctor_with_generation(self): - BLOB_NAME = "blob-name" - GENERATION = 12345 - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket, generation=GENERATION) - self.assertEqual(blob.generation, GENERATION) - - def _set_properties_helper(self, kms_key_name=None): - import datetime - from google.cloud._helpers import UTC - from google.cloud._helpers import _RFC3339_MICROS - - now = datetime.datetime.utcnow().replace(tzinfo=UTC) - NOW = now.strftime(_RFC3339_MICROS) - BLOB_NAME = "blob-name" - GENERATION = 12345 - BLOB_ID = "name/{}/{}".format(BLOB_NAME, GENERATION) - SELF_LINK = "http://example.com/self/" - METAGENERATION = 23456 - SIZE = 12345 - MD5_HASH = "DEADBEEF" - MEDIA_LINK = "http://example.com/media/" - ENTITY = "project-owner-12345" - ENTITY_ID = "23456" - CRC32C = "FACE0DAC" - COMPONENT_COUNT = 2 - ETAG = "ETAG" - resource = { - "id": BLOB_ID, - "selfLink": SELF_LINK, - "generation": GENERATION, - "metageneration": METAGENERATION, - "contentType": "text/plain", - "timeCreated": NOW, - "updated": NOW, - "timeDeleted": NOW, - "storageClass": "NEARLINE", - "timeStorageClassUpdated": NOW, - "size": SIZE, - "md5Hash": MD5_HASH, - "mediaLink": MEDIA_LINK, - "contentEncoding": "gzip", - "contentDisposition": "inline", - "contentLanguage": "en-US", - "cacheControl": "private", - "metadata": {"foo": "Foo"}, - "owner": {"entity": ENTITY, "entityId": ENTITY_ID}, - "crc32c": CRC32C, - "componentCount": COMPONENT_COUNT, - "etag": ETAG, - } - - if kms_key_name is not None: - resource["kmsKeyName"] = kms_key_name - - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - - blob._set_properties(resource) - - self.assertEqual(blob.id, BLOB_ID) - self.assertEqual(blob.self_link, SELF_LINK) - self.assertEqual(blob.generation, GENERATION) - self.assertEqual(blob.metageneration, METAGENERATION) - self.assertEqual(blob.content_type, "text/plain") - self.assertEqual(blob.time_created, now) - self.assertEqual(blob.updated, now) - self.assertEqual(blob.time_deleted, now) - self.assertEqual(blob.storage_class, "NEARLINE") - self.assertEqual(blob.size, SIZE) - self.assertEqual(blob.md5_hash, MD5_HASH) - self.assertEqual(blob.media_link, MEDIA_LINK) - self.assertEqual(blob.content_encoding, "gzip") - self.assertEqual(blob.content_disposition, "inline") - self.assertEqual(blob.content_language, "en-US") - self.assertEqual(blob.cache_control, "private") - self.assertEqual(blob.metadata, {"foo": "Foo"}) - self.assertEqual(blob.owner, {"entity": ENTITY, "entityId": ENTITY_ID}) - self.assertEqual(blob.crc32c, CRC32C) - self.assertEqual(blob.component_count, COMPONENT_COUNT) - self.assertEqual(blob.etag, ETAG) - - if kms_key_name is not None: - self.assertEqual(blob.kms_key_name, kms_key_name) - else: - self.assertIsNone(blob.kms_key_name) - - def test__set_properties_wo_kms_key_name(self): - self._set_properties_helper() - - def test__set_properties_w_kms_key_name(self): - kms_resource = ( - "projects/test-project-123/" - "locations/us/" - "keyRings/test-ring/" - "cryptoKeys/test-key" - ) - self._set_properties_helper(kms_key_name=kms_resource) - - def test_chunk_size_ctor(self): - from google.cloud.storage.blob import Blob - - BLOB_NAME = "blob-name" - BUCKET = object() - chunk_size = 10 * Blob._CHUNK_SIZE_MULTIPLE - blob = self._make_one(BLOB_NAME, bucket=BUCKET, chunk_size=chunk_size) - self.assertEqual(blob._chunk_size, chunk_size) - - def test_chunk_size_getter(self): - BLOB_NAME = "blob-name" - BUCKET = object() - blob = self._make_one(BLOB_NAME, bucket=BUCKET) - self.assertIsNone(blob.chunk_size) - VALUE = object() - blob._chunk_size = VALUE - self.assertIs(blob.chunk_size, VALUE) - - def test_chunk_size_setter(self): - BLOB_NAME = "blob-name" - BUCKET = object() - blob = self._make_one(BLOB_NAME, bucket=BUCKET) - self.assertIsNone(blob._chunk_size) - blob._CHUNK_SIZE_MULTIPLE = 10 - blob.chunk_size = 20 - self.assertEqual(blob._chunk_size, 20) - - def test_chunk_size_setter_bad_value(self): - BLOB_NAME = "blob-name" - BUCKET = object() - blob = self._make_one(BLOB_NAME, bucket=BUCKET) - self.assertIsNone(blob._chunk_size) - blob._CHUNK_SIZE_MULTIPLE = 10 - with self.assertRaises(ValueError): - blob.chunk_size = 11 - - def test_acl_property(self): - from google.cloud.storage.acl import ObjectACL - - fake_bucket = _Bucket() - blob = self._make_one(u"name", bucket=fake_bucket) - acl = blob.acl - self.assertIsInstance(acl, ObjectACL) - self.assertIs(acl, blob._acl) - - def test_path_bad_bucket(self): - fake_bucket = object() - name = u"blob-name" - blob = self._make_one(name, bucket=fake_bucket) - self.assertRaises(AttributeError, getattr, blob, "path") - - def test_path_no_name(self): - bucket = _Bucket() - blob = self._make_one(u"", bucket=bucket) - self.assertRaises(ValueError, getattr, blob, "path") - - def test_path_normal(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertEqual(blob.path, "/b/name/o/%s" % BLOB_NAME) - - def test_path_w_slash_in_name(self): - BLOB_NAME = "parent/child" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertEqual(blob.path, "/b/name/o/parent%2Fchild") - - def test_path_with_non_ascii(self): - blob_name = u"Caf\xe9" - bucket = _Bucket() - blob = self._make_one(blob_name, bucket=bucket) - self.assertEqual(blob.path, "/b/name/o/Caf%C3%A9") - - def test_bucket_readonly_property(self): - blob_name = "BLOB" - bucket = _Bucket() - other = _Bucket() - blob = self._make_one(blob_name, bucket=bucket) - with self.assertRaises(AttributeError): - blob.bucket = other - - def test_client(self): - blob_name = "BLOB" - bucket = _Bucket() - blob = self._make_one(blob_name, bucket=bucket) - self.assertIs(blob.client, bucket.client) - - def test_user_project(self): - user_project = "user-project-123" - blob_name = "BLOB" - bucket = _Bucket(user_project=user_project) - blob = self._make_one(blob_name, bucket=bucket) - self.assertEqual(blob.user_project, user_project) - - def test__encryption_headers_wo_encryption_key(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - expected = {} - self.assertEqual(blob._encryption_headers(), expected) - - def test__encryption_headers_w_encryption_key(self): - key = b"aa426195405adee2c8081bb9e7e74b19" - header_key_value = "YWE0MjYxOTU0MDVhZGVlMmM4MDgxYmI5ZTdlNzRiMTk=" - header_key_hash_value = "V3Kwe46nKc3xLv96+iJ707YfZfFvlObta8TQcx2gpm0=" - BLOB_NAME = "blob-name" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=key) - expected = { - "X-Goog-Encryption-Algorithm": "AES256", - "X-Goog-Encryption-Key": header_key_value, - "X-Goog-Encryption-Key-Sha256": header_key_hash_value, - } - self.assertEqual(blob._encryption_headers(), expected) - - def test__query_params_default(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertEqual(blob._query_params, {}) - - def test__query_params_w_user_project(self): - user_project = "user-project-123" - BLOB_NAME = "BLOB" - bucket = _Bucket(user_project=user_project) - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertEqual(blob._query_params, {"userProject": user_project}) - - def test__query_params_w_generation(self): - generation = 123456 - BLOB_NAME = "BLOB" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket, generation=generation) - self.assertEqual(blob._query_params, {"generation": generation}) - - def test_public_url(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertEqual( - blob.public_url, "https://storage.googleapis.com/name/%s" % BLOB_NAME - ) - - def test_public_url_w_slash_in_name(self): - BLOB_NAME = "parent/child" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertEqual( - blob.public_url, "https://storage.googleapis.com/name/parent/child" - ) - - def test_public_url_w_tilde_in_name(self): - BLOB_NAME = "foo~bar" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertEqual(blob.public_url, "https://storage.googleapis.com/name/foo~bar") - - def test_public_url_with_non_ascii(self): - blob_name = u"winter \N{snowman}" - bucket = _Bucket() - blob = self._make_one(blob_name, bucket=bucket) - expected_url = "https://storage.googleapis.com/name/winter%20%E2%98%83" - self.assertEqual(blob.public_url, expected_url) - - def test_generate_signed_url_w_invalid_version(self): - BLOB_NAME = "blob-name" - EXPIRATION = "2014-10-16T20:34:37.000Z" - connection = _Connection() - client = _Client(connection) - bucket = _Bucket(client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - with self.assertRaises(ValueError): - blob.generate_signed_url(EXPIRATION, version="nonesuch") - - def _generate_signed_url_helper( - self, - version=None, - blob_name="blob-name", - api_access_endpoint=None, - method="GET", - content_md5=None, - content_type=None, - response_type=None, - response_disposition=None, - generation=None, - headers=None, - query_parameters=None, - credentials=None, - expiration=None, - encryption_key=None, - access_token=None, - service_account_email=None, - virtual_hosted_style=False, - bucket_bound_hostname=None, - scheme="http", - ): - from six.moves.urllib import parse - from google.cloud._helpers import UTC - from google.cloud.storage.blob import _API_ACCESS_ENDPOINT - from google.cloud.storage.blob import _get_encryption_headers - - api_access_endpoint = api_access_endpoint or _API_ACCESS_ENDPOINT - - delta = datetime.timedelta(hours=1) - - if expiration is None: - expiration = datetime.datetime.utcnow().replace(tzinfo=UTC) + delta - - connection = _Connection() - client = _Client(connection) - bucket = _Bucket(client) - blob = self._make_one(blob_name, bucket=bucket, encryption_key=encryption_key) - - if version is None: - effective_version = "v2" - else: - effective_version = version - - to_patch = "google.cloud.storage.blob.generate_signed_url_{}".format( - effective_version - ) - - with mock.patch(to_patch) as signer: - signed_uri = blob.generate_signed_url( - expiration=expiration, - api_access_endpoint=api_access_endpoint, - method=method, - credentials=credentials, - content_md5=content_md5, - content_type=content_type, - response_type=response_type, - response_disposition=response_disposition, - generation=generation, - headers=headers, - query_parameters=query_parameters, - version=version, - access_token=access_token, - service_account_email=service_account_email, - virtual_hosted_style=virtual_hosted_style, - bucket_bound_hostname=bucket_bound_hostname, - ) - - self.assertEqual(signed_uri, signer.return_value) - - if credentials is None: - expected_creds = _Connection.credentials - else: - expected_creds = credentials - - encoded_name = blob_name.encode("utf-8") - quoted_name = parse.quote(encoded_name, safe=b"/~") - - if virtual_hosted_style: - expected_api_access_endpoint = "https://{}.storage.googleapis.com".format( - bucket.name - ) - elif bucket_bound_hostname: - if ":" in bucket_bound_hostname: - expected_api_access_endpoint = bucket_bound_hostname - else: - expected_api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format( - scheme=scheme, bucket_bound_hostname=bucket_bound_hostname - ) - else: - expected_api_access_endpoint = api_access_endpoint - expected_resource = "/{}/{}".format(bucket.name, quoted_name) - - if virtual_hosted_style or bucket_bound_hostname: - expected_resource = "/{}".format(quoted_name) - - if encryption_key is not None: - expected_headers = headers or {} - if effective_version == "v2": - expected_headers["X-Goog-Encryption-Algorithm"] = "AES256" - else: - expected_headers.update(_get_encryption_headers(encryption_key)) - else: - expected_headers = headers - - expected_kwargs = { - "resource": expected_resource, - "expiration": expiration, - "api_access_endpoint": expected_api_access_endpoint, - "method": method.upper(), - "content_md5": content_md5, - "content_type": content_type, - "response_type": response_type, - "response_disposition": response_disposition, - "generation": generation, - "headers": expected_headers, - "query_parameters": query_parameters, - "access_token": access_token, - "service_account_email": service_account_email, - } - signer.assert_called_once_with(expected_creds, **expected_kwargs) - - def test_generate_signed_url_no_version_passed_warning(self): - self._generate_signed_url_helper() - - def _generate_signed_url_v2_helper(self, **kw): - version = "v2" - self._generate_signed_url_helper(version, **kw) - - def test_generate_signed_url_v2_w_defaults(self): - self._generate_signed_url_v2_helper() - - def test_generate_signed_url_v2_w_expiration(self): - from google.cloud._helpers import UTC - - expiration = datetime.datetime.utcnow().replace(tzinfo=UTC) - self._generate_signed_url_v2_helper(expiration=expiration) - - def test_generate_signed_url_v2_w_non_ascii_name(self): - BLOB_NAME = u"\u0410\u043a\u043a\u043e\u0440\u0434\u044b.txt" - self._generate_signed_url_v2_helper(blob_name=BLOB_NAME) - - def test_generate_signed_url_v2_w_slash_in_name(self): - BLOB_NAME = "parent/child" - self._generate_signed_url_v2_helper(blob_name=BLOB_NAME) - - def test_generate_signed_url_v2_w_tilde_in_name(self): - BLOB_NAME = "foo~bar" - self._generate_signed_url_v2_helper(blob_name=BLOB_NAME) - - def test_generate_signed_url_v2_w_endpoint(self): - self._generate_signed_url_v2_helper( - api_access_endpoint="https://api.example.com/v1" - ) - - def test_generate_signed_url_v2_w_method(self): - self._generate_signed_url_v2_helper(method="POST") - - def test_generate_signed_url_v2_w_lowercase_method(self): - self._generate_signed_url_v2_helper(method="get") - - def test_generate_signed_url_v2_w_content_md5(self): - self._generate_signed_url_v2_helper(content_md5="FACEDACE") - - def test_generate_signed_url_v2_w_content_type(self): - self._generate_signed_url_v2_helper(content_type="text.html") - - def test_generate_signed_url_v2_w_response_type(self): - self._generate_signed_url_v2_helper(response_type="text.html") - - def test_generate_signed_url_v2_w_response_disposition(self): - self._generate_signed_url_v2_helper(response_disposition="inline") - - def test_generate_signed_url_v2_w_generation(self): - self._generate_signed_url_v2_helper(generation=12345) - - def test_generate_signed_url_v2_w_headers(self): - self._generate_signed_url_v2_helper(headers={"x-goog-foo": "bar"}) - - def test_generate_signed_url_v2_w_csek(self): - self._generate_signed_url_v2_helper(encryption_key=os.urandom(32)) - - def test_generate_signed_url_v2_w_csek_and_headers(self): - self._generate_signed_url_v2_helper( - encryption_key=os.urandom(32), headers={"x-goog-foo": "bar"} - ) - - def test_generate_signed_url_v2_w_credentials(self): - credentials = object() - self._generate_signed_url_v2_helper(credentials=credentials) - - def _generate_signed_url_v4_helper(self, **kw): - version = "v4" - self._generate_signed_url_helper(version, **kw) - - def test_generate_signed_url_v4_w_defaults(self): - self._generate_signed_url_v4_helper() - - def test_generate_signed_url_v4_w_non_ascii_name(self): - BLOB_NAME = u"\u0410\u043a\u043a\u043e\u0440\u0434\u044b.txt" - self._generate_signed_url_v4_helper(blob_name=BLOB_NAME) - - def test_generate_signed_url_v4_w_slash_in_name(self): - BLOB_NAME = "parent/child" - self._generate_signed_url_v4_helper(blob_name=BLOB_NAME) - - def test_generate_signed_url_v4_w_tilde_in_name(self): - BLOB_NAME = "foo~bar" - self._generate_signed_url_v4_helper(blob_name=BLOB_NAME) - - def test_generate_signed_url_v4_w_endpoint(self): - self._generate_signed_url_v4_helper( - api_access_endpoint="https://api.example.com/v1" - ) - - def test_generate_signed_url_v4_w_method(self): - self._generate_signed_url_v4_helper(method="POST") - - def test_generate_signed_url_v4_w_lowercase_method(self): - self._generate_signed_url_v4_helper(method="get") - - def test_generate_signed_url_v4_w_content_md5(self): - self._generate_signed_url_v4_helper(content_md5="FACEDACE") - - def test_generate_signed_url_v4_w_content_type(self): - self._generate_signed_url_v4_helper(content_type="text.html") - - def test_generate_signed_url_v4_w_response_type(self): - self._generate_signed_url_v4_helper(response_type="text.html") - - def test_generate_signed_url_v4_w_response_disposition(self): - self._generate_signed_url_v4_helper(response_disposition="inline") - - def test_generate_signed_url_v4_w_generation(self): - self._generate_signed_url_v4_helper(generation=12345) - - def test_generate_signed_url_v4_w_headers(self): - self._generate_signed_url_v4_helper(headers={"x-goog-foo": "bar"}) - - def test_generate_signed_url_v4_w_csek(self): - self._generate_signed_url_v4_helper(encryption_key=os.urandom(32)) - - def test_generate_signed_url_v4_w_csek_and_headers(self): - self._generate_signed_url_v4_helper( - encryption_key=os.urandom(32), headers={"x-goog-foo": "bar"} - ) - - def test_generate_signed_url_v4_w_virtual_hostname(self): - self._generate_signed_url_v4_helper(virtual_hosted_style=True) - - def test_generate_signed_url_v4_w_bucket_bound_hostname_w_scheme(self): - self._generate_signed_url_v4_helper( - bucket_bound_hostname="http://cdn.example.com" - ) - - def test_generate_signed_url_v4_w_bucket_bound_hostname_w_bare_hostname(self): - self._generate_signed_url_v4_helper(bucket_bound_hostname="cdn.example.com") - - def test_generate_signed_url_v4_w_credentials(self): - credentials = object() - self._generate_signed_url_v4_helper(credentials=credentials) - - def test_exists_miss(self): - NONESUCH = "nonesuch" - not_found_response = ({"status": http_client.NOT_FOUND}, b"") - connection = _Connection(not_found_response) - client = _Client(connection) - bucket = _Bucket(client) - blob = self._make_one(NONESUCH, bucket=bucket) - self.assertFalse(blob.exists(timeout=42)) - self.assertEqual(len(connection._requested), 1) - self.assertEqual( - connection._requested[0], - { - "method": "GET", - "path": "/b/name/o/{}".format(NONESUCH), - "query_params": {"fields": "name"}, - "_target_object": None, - "timeout": 42, - }, - ) - - def test_exists_hit_w_user_project(self): - BLOB_NAME = "blob-name" - USER_PROJECT = "user-project-123" - found_response = ({"status": http_client.OK}, b"") - connection = _Connection(found_response) - client = _Client(connection) - bucket = _Bucket(client, user_project=USER_PROJECT) - blob = self._make_one(BLOB_NAME, bucket=bucket) - bucket._blobs[BLOB_NAME] = 1 - self.assertTrue(blob.exists()) - self.assertEqual(len(connection._requested), 1) - self.assertEqual( - connection._requested[0], - { - "method": "GET", - "path": "/b/name/o/{}".format(BLOB_NAME), - "query_params": {"fields": "name", "userProject": USER_PROJECT}, - "_target_object": None, - "timeout": self._get_default_timeout(), - }, - ) - - def test_exists_hit_w_generation(self): - BLOB_NAME = "blob-name" - GENERATION = 123456 - found_response = ({"status": http_client.OK}, b"") - connection = _Connection(found_response) - client = _Client(connection) - bucket = _Bucket(client) - blob = self._make_one(BLOB_NAME, bucket=bucket, generation=GENERATION) - bucket._blobs[BLOB_NAME] = 1 - self.assertTrue(blob.exists()) - self.assertEqual(len(connection._requested), 1) - self.assertEqual( - connection._requested[0], - { - "method": "GET", - "path": "/b/name/o/{}".format(BLOB_NAME), - "query_params": {"fields": "name", "generation": GENERATION}, - "_target_object": None, - "timeout": self._get_default_timeout(), - }, - ) - - def test_delete_wo_generation(self): - BLOB_NAME = "blob-name" - not_found_response = ({"status": http_client.NOT_FOUND}, b"") - connection = _Connection(not_found_response) - client = _Client(connection) - bucket = _Bucket(client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - bucket._blobs[BLOB_NAME] = 1 - blob.delete() - self.assertFalse(blob.exists()) - self.assertEqual( - bucket._deleted, [(BLOB_NAME, None, None, self._get_default_timeout())] - ) - - def test_delete_w_generation(self): - BLOB_NAME = "blob-name" - GENERATION = 123456 - not_found_response = ({"status": http_client.NOT_FOUND}, b"") - connection = _Connection(not_found_response) - client = _Client(connection) - bucket = _Bucket(client) - blob = self._make_one(BLOB_NAME, bucket=bucket, generation=GENERATION) - bucket._blobs[BLOB_NAME] = 1 - blob.delete(timeout=42) - self.assertFalse(blob.exists()) - self.assertEqual(bucket._deleted, [(BLOB_NAME, None, GENERATION, 42)]) - - def test__get_transport(self): - client = mock.Mock(spec=[u"_credentials", "_http"]) - client._http = mock.sentinel.transport - blob = self._make_one(u"blob-name", bucket=None) - - transport = blob._get_transport(client) - - self.assertIs(transport, mock.sentinel.transport) - - def test__get_download_url_with_media_link(self): - blob_name = "something.txt" - bucket = _Bucket(name="IRRELEVANT") - blob = self._make_one(blob_name, bucket=bucket) - media_link = "http://test.invalid" - # Set the media link on the blob - blob._properties["mediaLink"] = media_link - - download_url = blob._get_download_url() - self.assertEqual(download_url, media_link) - - def test__get_download_url_with_media_link_w_user_project(self): - blob_name = "something.txt" - user_project = "user-project-123" - bucket = _Bucket(name="IRRELEVANT", user_project=user_project) - blob = self._make_one(blob_name, bucket=bucket) - media_link = "http://test.invalid" - # Set the media link on the blob - blob._properties["mediaLink"] = media_link - - download_url = blob._get_download_url() - self.assertEqual( - download_url, "{}?userProject={}".format(media_link, user_project) - ) - - def test__get_download_url_on_the_fly(self): - blob_name = "bzzz-fly.txt" - bucket = _Bucket(name="buhkit") - blob = self._make_one(blob_name, bucket=bucket) - - self.assertIsNone(blob.media_link) - download_url = blob._get_download_url() - expected_url = ( - "https://storage.googleapis.com/download/storage/v1/b/" - "buhkit/o/bzzz-fly.txt?alt=media" - ) - self.assertEqual(download_url, expected_url) - - def test__get_download_url_on_the_fly_with_generation(self): - blob_name = "pretend.txt" - bucket = _Bucket(name="fictional") - blob = self._make_one(blob_name, bucket=bucket) - generation = 1493058489532987 - # Set the media link on the blob - blob._properties["generation"] = str(generation) - - self.assertIsNone(blob.media_link) - download_url = blob._get_download_url() - expected_url = ( - "https://storage.googleapis.com/download/storage/v1/b/" - "fictional/o/pretend.txt?alt=media&generation=1493058489532987" - ) - self.assertEqual(download_url, expected_url) - - def test__get_download_url_on_the_fly_with_user_project(self): - blob_name = "pretend.txt" - user_project = "user-project-123" - bucket = _Bucket(name="fictional", user_project=user_project) - blob = self._make_one(blob_name, bucket=bucket) - - self.assertIsNone(blob.media_link) - download_url = blob._get_download_url() - expected_url = ( - "https://storage.googleapis.com/download/storage/v1/b/" - "fictional/o/pretend.txt?alt=media&userProject={}".format(user_project) - ) - self.assertEqual(download_url, expected_url) - - def test__get_download_url_on_the_fly_with_kms_key_name(self): - kms_resource = ( - "projects/test-project-123/" - "locations/us/" - "keyRings/test-ring/" - "cryptoKeys/test-key" - ) - blob_name = "bzzz-fly.txt" - bucket = _Bucket(name="buhkit") - blob = self._make_one(blob_name, bucket=bucket, kms_key_name=kms_resource) - - self.assertIsNone(blob.media_link) - download_url = blob._get_download_url() - expected_url = ( - "https://storage.googleapis.com/download/storage/v1/b/" - "buhkit/o/bzzz-fly.txt?alt=media" - ) - self.assertEqual(download_url, expected_url) - - @staticmethod - def _mock_requests_response(status_code, headers, content=b""): - import requests - - response = requests.Response() - response.status_code = status_code - response.headers.update(headers) - response.raw = None - response._content = content - - response.request = requests.Request("POST", "http://example.com").prepare() - return response - - def _do_download_helper_wo_chunks(self, w_range, raw_download): - blob_name = "blob-name" - client = mock.Mock() - bucket = _Bucket(client) - blob = self._make_one(blob_name, bucket=bucket) - self.assertIsNone(blob.chunk_size) - - transport = object() - file_obj = io.BytesIO() - download_url = "http://test.invalid" - headers = {} - - if raw_download: - patch = mock.patch("google.cloud.storage.blob.RawDownload") - else: - patch = mock.patch("google.cloud.storage.blob.Download") - - with patch as patched: - if w_range: - blob._do_download( - transport, - file_obj, - download_url, - headers, - start=1, - end=3, - raw_download=raw_download, - ) - else: - blob._do_download( - transport, - file_obj, - download_url, - headers, - raw_download=raw_download, - ) - - if w_range: - patched.assert_called_once_with( - download_url, stream=file_obj, headers=headers, start=1, end=3 - ) - else: - patched.assert_called_once_with( - download_url, stream=file_obj, headers=headers, start=None, end=None - ) - patched.return_value.consume.assert_called_once_with(transport) - - def test__do_download_wo_chunks_wo_range_wo_raw(self): - self._do_download_helper_wo_chunks(w_range=False, raw_download=False) - - def test__do_download_wo_chunks_w_range_wo_raw(self): - self._do_download_helper_wo_chunks(w_range=True, raw_download=False) - - def test__do_download_wo_chunks_wo_range_w_raw(self): - self._do_download_helper_wo_chunks(w_range=False, raw_download=True) - - def test__do_download_wo_chunks_w_range_w_raw(self): - self._do_download_helper_wo_chunks(w_range=True, raw_download=True) - - def _do_download_helper_w_chunks(self, w_range, raw_download): - blob_name = "blob-name" - client = mock.Mock(_credentials=_make_credentials(), spec=["_credentials"]) - bucket = _Bucket(client) - blob = self._make_one(blob_name, bucket=bucket) - blob._CHUNK_SIZE_MULTIPLE = 1 - chunk_size = blob.chunk_size = 3 - - transport = object() - file_obj = io.BytesIO() - download_url = "http://test.invalid" - headers = {} - - download = mock.Mock(finished=False, spec=["finished", "consume_next_chunk"]) - - def side_effect(_): - download.finished = True - - download.consume_next_chunk.side_effect = side_effect - - if raw_download: - patch = mock.patch("google.cloud.storage.blob.RawChunkedDownload") - else: - patch = mock.patch("google.cloud.storage.blob.ChunkedDownload") - - with patch as patched: - patched.return_value = download - if w_range: - blob._do_download( - transport, - file_obj, - download_url, - headers, - start=1, - end=3, - raw_download=raw_download, - ) - else: - blob._do_download( - transport, - file_obj, - download_url, - headers, - raw_download=raw_download, - ) - - if w_range: - patched.assert_called_once_with( - download_url, chunk_size, file_obj, headers=headers, start=1, end=3 - ) - else: - patched.assert_called_once_with( - download_url, chunk_size, file_obj, headers=headers, start=0, end=None - ) - download.consume_next_chunk.assert_called_once_with(transport) - - def test__do_download_w_chunks_wo_range_wo_raw(self): - self._do_download_helper_w_chunks(w_range=False, raw_download=False) - - def test__do_download_w_chunks_w_range_wo_raw(self): - self._do_download_helper_w_chunks(w_range=True, raw_download=False) - - def test__do_download_w_chunks_wo_range_w_raw(self): - self._do_download_helper_w_chunks(w_range=False, raw_download=True) - - def test__do_download_w_chunks_w_range_w_raw(self): - self._do_download_helper_w_chunks(w_range=True, raw_download=True) - - def test_download_to_file_with_failure(self): - import requests - from google.resumable_media import InvalidResponse - from google.cloud import exceptions - - raw_response = requests.Response() - raw_response.status_code = http_client.NOT_FOUND - raw_request = requests.Request("GET", "http://example.com") - raw_response.request = raw_request.prepare() - grmp_response = InvalidResponse(raw_response) - - blob_name = "blob-name" - media_link = "http://test.invalid" - client = mock.Mock(spec=[u"_http"]) - bucket = _Bucket(client) - blob = self._make_one(blob_name, bucket=bucket) - blob._properties["mediaLink"] = media_link - blob._do_download = mock.Mock() - blob._do_download.side_effect = grmp_response - - file_obj = io.BytesIO() - with self.assertRaises(exceptions.NotFound): - blob.download_to_file(file_obj) - - self.assertEqual(file_obj.tell(), 0) - - headers = {"accept-encoding": "gzip"} - blob._do_download.assert_called_once_with( - client._http, file_obj, media_link, headers, None, None, False - ) - - def test_download_to_file_wo_media_link(self): - blob_name = "blob-name" - client = mock.Mock(spec=[u"_http"]) - bucket = _Bucket(client) - blob = self._make_one(blob_name, bucket=bucket) - blob._do_download = mock.Mock() - file_obj = io.BytesIO() - - blob.download_to_file(file_obj) - - # Make sure the media link is still unknown. - self.assertIsNone(blob.media_link) - - expected_url = ( - "https://storage.googleapis.com/download/storage/v1/b/" - "name/o/blob-name?alt=media" - ) - headers = {"accept-encoding": "gzip"} - blob._do_download.assert_called_once_with( - client._http, file_obj, expected_url, headers, None, None, False - ) - - def _download_to_file_helper(self, use_chunks, raw_download): - blob_name = "blob-name" - client = mock.Mock(spec=[u"_http"]) - bucket = _Bucket(client) - media_link = "http://example.com/media/" - properties = {"mediaLink": media_link} - blob = self._make_one(blob_name, bucket=bucket, properties=properties) - if use_chunks: - blob._CHUNK_SIZE_MULTIPLE = 1 - blob.chunk_size = 3 - blob._do_download = mock.Mock() - - file_obj = io.BytesIO() - if raw_download: - blob.download_to_file(file_obj, raw_download=True) - else: - blob.download_to_file(file_obj) - - headers = {"accept-encoding": "gzip"} - blob._do_download.assert_called_once_with( - client._http, file_obj, media_link, headers, None, None, raw_download - ) - - def test_download_to_file_wo_chunks_wo_raw(self): - self._download_to_file_helper(use_chunks=False, raw_download=False) - - def test_download_to_file_w_chunks_wo_raw(self): - self._download_to_file_helper(use_chunks=True, raw_download=False) - - def test_download_to_file_wo_chunks_w_raw(self): - self._download_to_file_helper(use_chunks=False, raw_download=True) - - def test_download_to_file_w_chunks_w_raw(self): - self._download_to_file_helper(use_chunks=True, raw_download=True) - - def _download_to_filename_helper(self, updated, raw_download): - import os - import time - from google.cloud._testing import _NamedTemporaryFile - - blob_name = "blob-name" - client = mock.Mock(spec=["_http"]) - bucket = _Bucket(client) - media_link = "http://example.com/media/" - properties = {"mediaLink": media_link} - if updated is not None: - properties["updated"] = updated - - blob = self._make_one(blob_name, bucket=bucket, properties=properties) - blob._do_download = mock.Mock() - - with _NamedTemporaryFile() as temp: - blob.download_to_filename(temp.name, raw_download=raw_download) - if updated is None: - self.assertIsNone(blob.updated) - else: - mtime = os.path.getmtime(temp.name) - updated_time = time.mktime(blob.updated.timetuple()) - self.assertEqual(mtime, updated_time) - - headers = {"accept-encoding": "gzip"} - blob._do_download.assert_called_once_with( - client._http, mock.ANY, media_link, headers, None, None, raw_download - ) - stream = blob._do_download.mock_calls[0].args[1] - self.assertEqual(stream.name, temp.name) - - def test_download_to_filename_w_updated_wo_raw(self): - updated = "2014-12-06T13:13:50.690Z" - self._download_to_filename_helper(updated=updated, raw_download=False) - - def test_download_to_filename_wo_updated_wo_raw(self): - self._download_to_filename_helper(updated=None, raw_download=False) - - def test_download_to_filename_w_updated_w_raw(self): - updated = "2014-12-06T13:13:50.690Z" - self._download_to_filename_helper(updated=updated, raw_download=True) - - def test_download_to_filename_wo_updated_w_raw(self): - self._download_to_filename_helper(updated=None, raw_download=True) - - def test_download_to_filename_corrupted(self): - from google.resumable_media import DataCorruption - - blob_name = "blob-name" - client = mock.Mock(spec=["_http"]) - bucket = _Bucket(client) - media_link = "http://example.com/media/" - properties = {"mediaLink": media_link} - - blob = self._make_one(blob_name, bucket=bucket, properties=properties) - blob._do_download = mock.Mock() - blob._do_download.side_effect = DataCorruption("testing") - - # Try to download into a temporary file (don't use - # `_NamedTemporaryFile` it will try to remove after the file is - # already removed) - filehandle, filename = tempfile.mkstemp() - os.close(filehandle) - self.assertTrue(os.path.exists(filename)) - - with self.assertRaises(DataCorruption): - blob.download_to_filename(filename) - - # Make sure the file was cleaned up. - self.assertFalse(os.path.exists(filename)) - - headers = {"accept-encoding": "gzip"} - blob._do_download.assert_called_once_with( - client._http, mock.ANY, media_link, headers, None, None, False - ) - stream = blob._do_download.mock_calls[0].args[1] - self.assertEqual(stream.name, filename) - - def test_download_to_filename_w_key(self): - from google.cloud._testing import _NamedTemporaryFile - from google.cloud.storage.blob import _get_encryption_headers - - blob_name = "blob-name" - # Create a fake client/bucket and use them in the Blob() constructor. - client = mock.Mock(spec=["_http"]) - bucket = _Bucket(client) - media_link = "http://example.com/media/" - properties = {"mediaLink": media_link} - key = b"aa426195405adee2c8081bb9e7e74b19" - blob = self._make_one( - blob_name, bucket=bucket, properties=properties, encryption_key=key - ) - blob._do_download = mock.Mock() - - with _NamedTemporaryFile() as temp: - blob.download_to_filename(temp.name) - - headers = {"accept-encoding": "gzip"} - headers.update(_get_encryption_headers(key)) - blob._do_download.assert_called_once_with( - client._http, mock.ANY, media_link, headers, None, None, False - ) - stream = blob._do_download.mock_calls[0].args[1] - self.assertEqual(stream.name, temp.name) - - def _download_as_string_helper(self, raw_download): - blob_name = "blob-name" - client = mock.Mock(spec=["_http"]) - bucket = _Bucket(client) - media_link = "http://example.com/media/" - properties = {"mediaLink": media_link} - blob = self._make_one(blob_name, bucket=bucket, properties=properties) - blob._do_download = mock.Mock() - - fetched = blob.download_as_string(raw_download=raw_download) - self.assertEqual(fetched, b"") - - headers = {"accept-encoding": "gzip"} - blob._do_download.assert_called_once_with( - client._http, mock.ANY, media_link, headers, None, None, raw_download - ) - stream = blob._do_download.mock_calls[0].args[1] - self.assertIsInstance(stream, io.BytesIO) - - def test_download_as_string_wo_raw(self): - self._download_as_string_helper(raw_download=False) - - def test_download_as_string_w_raw(self): - self._download_as_string_helper(raw_download=True) - - def test__get_content_type_explicit(self): - blob = self._make_one(u"blob-name", bucket=None) - - content_type = u"text/plain" - return_value = blob._get_content_type(content_type) - self.assertEqual(return_value, content_type) - - def test__get_content_type_from_blob(self): - blob = self._make_one(u"blob-name", bucket=None) - blob.content_type = u"video/mp4" - - return_value = blob._get_content_type(None) - self.assertEqual(return_value, blob.content_type) - - def test__get_content_type_from_filename(self): - blob = self._make_one(u"blob-name", bucket=None) - - return_value = blob._get_content_type(None, filename="archive.tar") - self.assertEqual(return_value, "application/x-tar") - - def test__get_content_type_default(self): - blob = self._make_one(u"blob-name", bucket=None) - - return_value = blob._get_content_type(None) - self.assertEqual(return_value, u"application/octet-stream") - - def test__get_writable_metadata_no_changes(self): - name = u"blob-name" - blob = self._make_one(name, bucket=None) - - object_metadata = blob._get_writable_metadata() - expected = {"name": name} - self.assertEqual(object_metadata, expected) - - def test__get_writable_metadata_with_changes(self): - name = u"blob-name" - blob = self._make_one(name, bucket=None) - blob.storage_class = "NEARLINE" - blob.cache_control = "max-age=3600" - blob.metadata = {"color": "red"} - - object_metadata = blob._get_writable_metadata() - expected = { - "cacheControl": blob.cache_control, - "metadata": blob.metadata, - "name": name, - "storageClass": blob.storage_class, - } - self.assertEqual(object_metadata, expected) - - def test__get_writable_metadata_unwritable_field(self): - name = u"blob-name" - properties = {"updated": "2016-10-16T18:18:18.181Z"} - blob = self._make_one(name, bucket=None, properties=properties) - # Fake that `updated` is in changes. - blob._changes.add("updated") - - object_metadata = blob._get_writable_metadata() - expected = {"name": name} - self.assertEqual(object_metadata, expected) - - def test__set_metadata_to_none(self): - name = u"blob-name" - blob = self._make_one(name, bucket=None) - blob.storage_class = "NEARLINE" - blob.cache_control = "max-age=3600" - - with mock.patch("google.cloud.storage.blob.Blob._patch_property") as patch_prop: - blob.metadata = None - patch_prop.assert_called_once_with("metadata", None) - - def test__get_upload_arguments(self): - name = u"blob-name" - key = b"[pXw@,p@@AfBfrR3x-2b2SCHR,.?YwRO" - blob = self._make_one(name, bucket=None, encryption_key=key) - blob.content_disposition = "inline" - - content_type = u"image/jpeg" - info = blob._get_upload_arguments(content_type) - - headers, object_metadata, new_content_type = info - header_key_value = "W3BYd0AscEBAQWZCZnJSM3gtMmIyU0NIUiwuP1l3Uk8=" - header_key_hash_value = "G0++dxF4q5rG4o9kE8gvEKn15RH6wLm0wXV1MgAlXOg=" - expected_headers = { - "X-Goog-Encryption-Algorithm": "AES256", - "X-Goog-Encryption-Key": header_key_value, - "X-Goog-Encryption-Key-Sha256": header_key_hash_value, - } - self.assertEqual(headers, expected_headers) - expected_metadata = { - "contentDisposition": blob.content_disposition, - "name": name, - } - self.assertEqual(object_metadata, expected_metadata) - self.assertEqual(new_content_type, content_type) - - def _mock_transport(self, status_code, headers, content=b""): - fake_transport = mock.Mock(spec=["request"]) - fake_response = self._mock_requests_response( - status_code, headers, content=content - ) - fake_transport.request.return_value = fake_response - return fake_transport - - def _do_multipart_success( - self, - mock_get_boundary, - size=None, - num_retries=None, - user_project=None, - predefined_acl=None, - kms_key_name=None, - ): - from six.moves.urllib.parse import urlencode - - bucket = _Bucket(name="w00t", user_project=user_project) - blob = self._make_one(u"blob-name", bucket=bucket, kms_key_name=kms_key_name) - self.assertIsNone(blob.chunk_size) - - # Create mocks to be checked for doing transport. - transport = self._mock_transport(http_client.OK, {}) - - # Create some mock arguments. - client = mock.Mock(_http=transport, spec=["_http"]) - data = b"data here hear hier" - stream = io.BytesIO(data) - content_type = u"application/xml" - response = blob._do_multipart_upload( - client, stream, content_type, size, num_retries, predefined_acl - ) - - # Check the mocks and the returned value. - self.assertIs(response, transport.request.return_value) - if size is None: - data_read = data - self.assertEqual(stream.tell(), len(data)) - else: - data_read = data[:size] - self.assertEqual(stream.tell(), size) - - mock_get_boundary.assert_called_once_with() - - upload_url = ( - "https://storage.googleapis.com/upload/storage/v1" + bucket.path + "/o" - ) - - qs_params = [("uploadType", "multipart")] - - if user_project is not None: - qs_params.append(("userProject", user_project)) - - if predefined_acl is not None: - qs_params.append(("predefinedAcl", predefined_acl)) - - if kms_key_name is not None: - qs_params.append(("kmsKeyName", kms_key_name)) - - upload_url += "?" + urlencode(qs_params) - - payload = ( - b"--==0==\r\n" - + b"content-type: application/json; charset=UTF-8\r\n\r\n" - + b'{"name": "blob-name"}\r\n' - + b"--==0==\r\n" - + b"content-type: application/xml\r\n\r\n" - + data_read - + b"\r\n--==0==--" - ) - headers = {"content-type": b'multipart/related; boundary="==0=="'} - transport.request.assert_called_once_with( - "POST", upload_url, data=payload, headers=headers, timeout=mock.ANY - ) - - @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") - def test__do_multipart_upload_no_size(self, mock_get_boundary): - self._do_multipart_success(mock_get_boundary, predefined_acl="private") - - @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") - def test__do_multipart_upload_with_size(self, mock_get_boundary): - self._do_multipart_success(mock_get_boundary, size=10) - - @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") - def test__do_multipart_upload_with_user_project(self, mock_get_boundary): - user_project = "user-project-123" - self._do_multipart_success(mock_get_boundary, user_project=user_project) - - @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") - def test__do_multipart_upload_with_kms(self, mock_get_boundary): - kms_resource = ( - "projects/test-project-123/" - "locations/us/" - "keyRings/test-ring/" - "cryptoKeys/test-key" - ) - self._do_multipart_success(mock_get_boundary, kms_key_name=kms_resource) - - @mock.patch(u"google.resumable_media._upload.get_boundary", return_value=b"==0==") - def test__do_multipart_upload_with_retry(self, mock_get_boundary): - self._do_multipart_success(mock_get_boundary, num_retries=8) - - def test__do_multipart_upload_bad_size(self): - blob = self._make_one(u"blob-name", bucket=None) - - data = b"data here hear hier" - stream = io.BytesIO(data) - size = 50 - self.assertGreater(size, len(data)) - - with self.assertRaises(ValueError) as exc_info: - blob._do_multipart_upload(None, stream, None, size, None, None) - - exc_contents = str(exc_info.exception) - self.assertIn("was specified but the file-like object only had", exc_contents) - self.assertEqual(stream.tell(), len(data)) - - def _initiate_resumable_helper( - self, - size=None, - extra_headers=None, - chunk_size=None, - num_retries=None, - user_project=None, - predefined_acl=None, - blob_chunk_size=786432, - kms_key_name=None, - ): - from six.moves.urllib.parse import urlencode - from google.resumable_media.requests import ResumableUpload - from google.cloud.storage.blob import _DEFAULT_CHUNKSIZE - - bucket = _Bucket(name="whammy", user_project=user_project) - blob = self._make_one(u"blob-name", bucket=bucket, kms_key_name=kms_key_name) - blob.metadata = {"rook": "takes knight"} - blob.chunk_size = blob_chunk_size - if blob_chunk_size is not None: - self.assertIsNotNone(blob.chunk_size) - else: - self.assertIsNone(blob.chunk_size) - - # Need to make sure **same** dict is used because ``json.dumps()`` - # will depend on the hash order. - object_metadata = blob._get_writable_metadata() - blob._get_writable_metadata = mock.Mock(return_value=object_metadata, spec=[]) - - # Create mocks to be checked for doing transport. - resumable_url = "http://test.invalid?upload_id=hey-you" - response_headers = {"location": resumable_url} - transport = self._mock_transport(http_client.OK, response_headers) - - # Create some mock arguments and call the method under test. - client = mock.Mock(_http=transport, spec=[u"_http"]) - data = b"hello hallo halo hi-low" - stream = io.BytesIO(data) - content_type = u"text/plain" - upload, transport = blob._initiate_resumable_upload( - client, - stream, - content_type, - size, - num_retries, - extra_headers=extra_headers, - chunk_size=chunk_size, - predefined_acl=predefined_acl, - ) - - # Check the returned values. - self.assertIsInstance(upload, ResumableUpload) - - upload_url = ( - "https://storage.googleapis.com/upload/storage/v1" + bucket.path + "/o" - ) - qs_params = [("uploadType", "resumable")] - - if user_project is not None: - qs_params.append(("userProject", user_project)) - - if predefined_acl is not None: - qs_params.append(("predefinedAcl", predefined_acl)) - - if kms_key_name is not None: - qs_params.append(("kmsKeyName", kms_key_name)) - - upload_url += "?" + urlencode(qs_params) - - self.assertEqual(upload.upload_url, upload_url) - if extra_headers is None: - self.assertEqual(upload._headers, {}) - else: - self.assertEqual(upload._headers, extra_headers) - self.assertIsNot(upload._headers, extra_headers) - self.assertFalse(upload.finished) - if chunk_size is None: - if blob_chunk_size is None: - self.assertEqual(upload._chunk_size, _DEFAULT_CHUNKSIZE) - else: - self.assertEqual(upload._chunk_size, blob.chunk_size) - else: - self.assertNotEqual(blob.chunk_size, chunk_size) - self.assertEqual(upload._chunk_size, chunk_size) - self.assertIs(upload._stream, stream) - if size is None: - self.assertIsNone(upload._total_bytes) - else: - self.assertEqual(upload._total_bytes, size) - self.assertEqual(upload._content_type, content_type) - self.assertEqual(upload.resumable_url, resumable_url) - retry_strategy = upload._retry_strategy - self.assertEqual(retry_strategy.max_sleep, 64.0) - if num_retries is None: - self.assertEqual(retry_strategy.max_cumulative_retry, 600.0) - self.assertIsNone(retry_strategy.max_retries) - else: - self.assertIsNone(retry_strategy.max_cumulative_retry) - self.assertEqual(retry_strategy.max_retries, num_retries) - self.assertIs(transport, transport) - # Make sure we never read from the stream. - self.assertEqual(stream.tell(), 0) - - # Check the mocks. - blob._get_writable_metadata.assert_called_once_with() - payload = json.dumps(object_metadata).encode("utf-8") - expected_headers = { - "content-type": "application/json; charset=UTF-8", - "x-upload-content-type": content_type, - } - if size is not None: - expected_headers["x-upload-content-length"] = str(size) - if extra_headers is not None: - expected_headers.update(extra_headers) - transport.request.assert_called_once_with( - "POST", upload_url, data=payload, headers=expected_headers, timeout=mock.ANY - ) - - def test__initiate_resumable_upload_no_size(self): - self._initiate_resumable_helper() - - def test__initiate_resumable_upload_with_size(self): - self._initiate_resumable_helper(size=10000) - - def test__initiate_resumable_upload_with_user_project(self): - user_project = "user-project-123" - self._initiate_resumable_helper(user_project=user_project) - - def test__initiate_resumable_upload_with_kms(self): - kms_resource = ( - "projects/test-project-123/" - "locations/us/" - "keyRings/test-ring/" - "cryptoKeys/test-key" - ) - self._initiate_resumable_helper(kms_key_name=kms_resource) - - def test__initiate_resumable_upload_without_chunk_size(self): - self._initiate_resumable_helper(blob_chunk_size=None) - - def test__initiate_resumable_upload_with_chunk_size(self): - one_mb = 1048576 - self._initiate_resumable_helper(chunk_size=one_mb) - - def test__initiate_resumable_upload_with_extra_headers(self): - extra_headers = {"origin": "http://not-in-kansas-anymore.invalid"} - self._initiate_resumable_helper(extra_headers=extra_headers) - - def test__initiate_resumable_upload_with_retry(self): - self._initiate_resumable_helper(num_retries=11) - - def test__initiate_resumable_upload_with_predefined_acl(self): - self._initiate_resumable_helper(predefined_acl="private") - - def _make_resumable_transport(self, headers1, headers2, headers3, total_bytes): - from google import resumable_media - - fake_transport = mock.Mock(spec=["request"]) - - fake_response1 = self._mock_requests_response(http_client.OK, headers1) - fake_response2 = self._mock_requests_response( - resumable_media.PERMANENT_REDIRECT, headers2 - ) - json_body = '{{"size": "{:d}"}}'.format(total_bytes) - fake_response3 = self._mock_requests_response( - http_client.OK, headers3, content=json_body.encode("utf-8") - ) - - responses = [fake_response1, fake_response2, fake_response3] - fake_transport.request.side_effect = responses - return fake_transport, responses - - @staticmethod - def _do_resumable_upload_call0(blob, content_type, size=None, predefined_acl=None): - # First mock transport.request() does initiates upload. - upload_url = ( - "https://storage.googleapis.com/upload/storage/v1" - + blob.bucket.path - + "/o?uploadType=resumable" - ) - if predefined_acl is not None: - upload_url += "&predefinedAcl={}".format(predefined_acl) - expected_headers = { - "content-type": "application/json; charset=UTF-8", - "x-upload-content-type": content_type, - } - if size is not None: - expected_headers["x-upload-content-length"] = str(size) - payload = json.dumps({"name": blob.name}).encode("utf-8") - return mock.call( - "POST", upload_url, data=payload, headers=expected_headers, timeout=mock.ANY - ) - - @staticmethod - def _do_resumable_upload_call1( - blob, content_type, data, resumable_url, size=None, predefined_acl=None - ): - # Second mock transport.request() does sends first chunk. - if size is None: - content_range = "bytes 0-{:d}/*".format(blob.chunk_size - 1) - else: - content_range = "bytes 0-{:d}/{:d}".format(blob.chunk_size - 1, size) - - expected_headers = { - "content-type": content_type, - "content-range": content_range, - } - payload = data[: blob.chunk_size] - return mock.call( - "PUT", - resumable_url, - data=payload, - headers=expected_headers, - timeout=mock.ANY, - ) - - @staticmethod - def _do_resumable_upload_call2( - blob, content_type, data, resumable_url, total_bytes, predefined_acl=None - ): - # Third mock transport.request() does sends last chunk. - content_range = "bytes {:d}-{:d}/{:d}".format( - blob.chunk_size, total_bytes - 1, total_bytes - ) - expected_headers = { - "content-type": content_type, - "content-range": content_range, - } - payload = data[blob.chunk_size :] - return mock.call( - "PUT", - resumable_url, - data=payload, - headers=expected_headers, - timeout=mock.ANY, - ) - - def _do_resumable_helper( - self, use_size=False, num_retries=None, predefined_acl=None - ): - bucket = _Bucket(name="yesterday") - blob = self._make_one(u"blob-name", bucket=bucket) - blob.chunk_size = blob._CHUNK_SIZE_MULTIPLE - self.assertIsNotNone(blob.chunk_size) - - # Data to be uploaded. - data = b"" + (b"A" * blob.chunk_size) + b"" - total_bytes = len(data) - if use_size: - size = total_bytes - else: - size = None - - # Create mocks to be checked for doing transport. - resumable_url = "http://test.invalid?upload_id=and-then-there-was-1" - headers1 = {"location": resumable_url} - headers2 = {"range": "bytes=0-{:d}".format(blob.chunk_size - 1)} - transport, responses = self._make_resumable_transport( - headers1, headers2, {}, total_bytes - ) - - # Create some mock arguments and call the method under test. - client = mock.Mock(_http=transport, spec=["_http"]) - stream = io.BytesIO(data) - content_type = u"text/html" - response = blob._do_resumable_upload( - client, stream, content_type, size, num_retries, predefined_acl - ) - - # Check the returned values. - self.assertIs(response, responses[2]) - self.assertEqual(stream.tell(), total_bytes) - - # Check the mocks. - call0 = self._do_resumable_upload_call0( - blob, content_type, size=size, predefined_acl=predefined_acl - ) - call1 = self._do_resumable_upload_call1( - blob, - content_type, - data, - resumable_url, - size=size, - predefined_acl=predefined_acl, - ) - call2 = self._do_resumable_upload_call2( - blob, - content_type, - data, - resumable_url, - total_bytes, - predefined_acl=predefined_acl, - ) - self.assertEqual(transport.request.mock_calls, [call0, call1, call2]) - - def test__do_resumable_upload_no_size(self): - self._do_resumable_helper() - - def test__do_resumable_upload_with_size(self): - self._do_resumable_helper(use_size=True) - - def test__do_resumable_upload_with_retry(self): - self._do_resumable_helper(num_retries=6) - - def test__do_resumable_upload_with_predefined_acl(self): - self._do_resumable_helper(predefined_acl="private") - - def _do_upload_helper( - self, chunk_size=None, num_retries=None, predefined_acl=None, size=None - ): - from google.cloud.storage.blob import _MAX_MULTIPART_SIZE - - blob = self._make_one(u"blob-name", bucket=None) - - # Create a fake response. - response = mock.Mock(spec=[u"json"]) - response.json.return_value = mock.sentinel.json - # Mock **both** helpers. - blob._do_multipart_upload = mock.Mock(return_value=response, spec=[]) - blob._do_resumable_upload = mock.Mock(return_value=response, spec=[]) - - if chunk_size is None: - self.assertIsNone(blob.chunk_size) - else: - blob.chunk_size = chunk_size - self.assertIsNotNone(blob.chunk_size) - - client = mock.sentinel.client - stream = mock.sentinel.stream - content_type = u"video/mp4" - if size is None: - size = 12345654321 - # Make the request and check the mocks. - created_json = blob._do_upload( - client, stream, content_type, size, num_retries, predefined_acl - ) - self.assertIs(created_json, mock.sentinel.json) - response.json.assert_called_once_with() - if size is not None and size <= _MAX_MULTIPART_SIZE: - blob._do_multipart_upload.assert_called_once_with( - client, stream, content_type, size, num_retries, predefined_acl - ) - blob._do_resumable_upload.assert_not_called() - else: - blob._do_multipart_upload.assert_not_called() - blob._do_resumable_upload.assert_called_once_with( - client, stream, content_type, size, num_retries, predefined_acl - ) - - def test__do_upload_uses_multipart(self): - from google.cloud.storage.blob import _MAX_MULTIPART_SIZE - - self._do_upload_helper(size=_MAX_MULTIPART_SIZE) - - def test__do_upload_uses_resumable(self): - from google.cloud.storage.blob import _MAX_MULTIPART_SIZE - - chunk_size = 256 * 1024 # 256KB - self._do_upload_helper(chunk_size=chunk_size, size=_MAX_MULTIPART_SIZE + 1) - - def test__do_upload_with_retry(self): - self._do_upload_helper(num_retries=20) - - def _upload_from_file_helper(self, side_effect=None, **kwargs): - from google.cloud._helpers import UTC - - blob = self._make_one("blob-name", bucket=None) - # Mock low-level upload helper on blob (it is tested elsewhere). - created_json = {"updated": "2017-01-01T09:09:09.081Z"} - blob._do_upload = mock.Mock(return_value=created_json, spec=[]) - if side_effect is not None: - blob._do_upload.side_effect = side_effect - # Make sure `updated` is empty before the request. - self.assertIsNone(blob.updated) - - data = b"data is here" - stream = io.BytesIO(data) - stream.seek(2) # Not at zero. - content_type = u"font/woff" - client = mock.sentinel.client - predefined_acl = kwargs.get("predefined_acl", None) - ret_val = blob.upload_from_file( - stream, size=len(data), content_type=content_type, client=client, **kwargs - ) - - # Check the response and side-effects. - self.assertIsNone(ret_val) - new_updated = datetime.datetime(2017, 1, 1, 9, 9, 9, 81000, tzinfo=UTC) - self.assertEqual(blob.updated, new_updated) - - # Check the mock. - num_retries = kwargs.get("num_retries") - blob._do_upload.assert_called_once_with( - client, stream, content_type, len(data), num_retries, predefined_acl - ) - return stream - - def test_upload_from_file_success(self): - stream = self._upload_from_file_helper(predefined_acl="private") - assert stream.tell() == 2 - - @mock.patch("warnings.warn") - def test_upload_from_file_with_retries(self, mock_warn): - from google.cloud.storage import blob as blob_module - - self._upload_from_file_helper(num_retries=20) - mock_warn.assert_called_once_with( - blob_module._NUM_RETRIES_MESSAGE, DeprecationWarning, stacklevel=2 - ) - - def test_upload_from_file_with_rewind(self): - stream = self._upload_from_file_helper(rewind=True) - assert stream.tell() == 0 - - def test_upload_from_file_failure(self): - import requests - - from google.resumable_media import InvalidResponse - from google.cloud import exceptions - - message = "Someone is already in this spot." - response = requests.Response() - response.status_code = http_client.CONFLICT - response.request = requests.Request("POST", "http://example.com").prepare() - side_effect = InvalidResponse(response, message) - - with self.assertRaises(exceptions.Conflict) as exc_info: - self._upload_from_file_helper(side_effect=side_effect) - - self.assertIn(message, exc_info.exception.message) - self.assertEqual(exc_info.exception.errors, []) - - def _do_upload_mock_call_helper(self, blob, client, content_type, size): - self.assertEqual(blob._do_upload.call_count, 1) - mock_call = blob._do_upload.mock_calls[0] - call_name, pos_args, kwargs = mock_call - self.assertEqual(call_name, "") - self.assertEqual(len(pos_args), 6) - self.assertEqual(pos_args[0], client) - self.assertEqual(pos_args[2], content_type) - self.assertEqual(pos_args[3], size) - self.assertIsNone(pos_args[4]) # num_retries - self.assertIsNone(pos_args[5]) # predefined_acl - self.assertEqual(kwargs, {}) - - return pos_args[1] - - def test_upload_from_filename(self): - from google.cloud._testing import _NamedTemporaryFile - - blob = self._make_one("blob-name", bucket=None) - # Mock low-level upload helper on blob (it is tested elsewhere). - created_json = {"metadata": {"mint": "ice-cream"}} - blob._do_upload = mock.Mock(return_value=created_json, spec=[]) - # Make sure `metadata` is empty before the request. - self.assertIsNone(blob.metadata) - - data = b"soooo much data" - content_type = u"image/svg+xml" - client = mock.sentinel.client - with _NamedTemporaryFile() as temp: - with open(temp.name, "wb") as file_obj: - file_obj.write(data) - - ret_val = blob.upload_from_filename( - temp.name, content_type=content_type, client=client - ) - - # Check the response and side-effects. - self.assertIsNone(ret_val) - self.assertEqual(blob.metadata, created_json["metadata"]) - - # Check the mock. - stream = self._do_upload_mock_call_helper(blob, client, content_type, len(data)) - self.assertTrue(stream.closed) - self.assertEqual(stream.mode, "rb") - self.assertEqual(stream.name, temp.name) - - def _upload_from_string_helper(self, data, **kwargs): - from google.cloud._helpers import _to_bytes - - blob = self._make_one("blob-name", bucket=None) - - # Mock low-level upload helper on blob (it is tested elsewhere). - created_json = {"componentCount": "5"} - blob._do_upload = mock.Mock(return_value=created_json, spec=[]) - # Make sure `metadata` is empty before the request. - self.assertIsNone(blob.component_count) - - client = mock.sentinel.client - ret_val = blob.upload_from_string(data, client=client, **kwargs) - - # Check the response and side-effects. - self.assertIsNone(ret_val) - self.assertEqual(blob.component_count, 5) - - # Check the mock. - payload = _to_bytes(data, encoding="utf-8") - stream = self._do_upload_mock_call_helper( - blob, client, "text/plain", len(payload) - ) - self.assertIsInstance(stream, io.BytesIO) - self.assertEqual(stream.getvalue(), payload) - - def test_upload_from_string_w_bytes(self): - data = b"XB]jb\xb8tad\xe0" - self._upload_from_string_helper(data) - - def test_upload_from_string_w_text(self): - data = u"\N{snowman} \N{sailboat}" - self._upload_from_string_helper(data) - - def _create_resumable_upload_session_helper(self, origin=None, side_effect=None): - bucket = _Bucket(name="alex-trebek") - blob = self._make_one("blob-name", bucket=bucket) - chunk_size = 99 * blob._CHUNK_SIZE_MULTIPLE - blob.chunk_size = chunk_size - - # Create mocks to be checked for doing transport. - resumable_url = "http://test.invalid?upload_id=clean-up-everybody" - response_headers = {"location": resumable_url} - transport = self._mock_transport(http_client.OK, response_headers) - if side_effect is not None: - transport.request.side_effect = side_effect - - # Create some mock arguments and call the method under test. - content_type = u"text/plain" - size = 10000 - client = mock.Mock(_http=transport, spec=[u"_http"]) - new_url = blob.create_resumable_upload_session( - content_type=content_type, size=size, origin=origin, client=client - ) - - # Check the returned value and (lack of) side-effect. - self.assertEqual(new_url, resumable_url) - self.assertEqual(blob.chunk_size, chunk_size) - - # Check the mocks. - upload_url = ( - "https://storage.googleapis.com/upload/storage/v1" - + bucket.path - + "/o?uploadType=resumable" - ) - payload = b'{"name": "blob-name"}' - expected_headers = { - "content-type": "application/json; charset=UTF-8", - "x-upload-content-length": str(size), - "x-upload-content-type": content_type, - } - if origin is not None: - expected_headers["Origin"] = origin - transport.request.assert_called_once_with( - "POST", upload_url, data=payload, headers=expected_headers, timeout=mock.ANY - ) - - def test_create_resumable_upload_session(self): - self._create_resumable_upload_session_helper() - - def test_create_resumable_upload_session_with_origin(self): - self._create_resumable_upload_session_helper(origin="http://google.com") - - def test_create_resumable_upload_session_with_failure(self): - from google.resumable_media import InvalidResponse - from google.cloud import exceptions - - message = "5-oh-3 woe is me." - response = self._mock_requests_response( - status_code=http_client.SERVICE_UNAVAILABLE, headers={} - ) - side_effect = InvalidResponse(response, message) - - with self.assertRaises(exceptions.ServiceUnavailable) as exc_info: - self._create_resumable_upload_session_helper(side_effect=side_effect) - - self.assertIn(message, exc_info.exception.message) - self.assertEqual(exc_info.exception.errors, []) - - def test_get_iam_policy(self): - from google.cloud.storage.iam import STORAGE_OWNER_ROLE - from google.cloud.storage.iam import STORAGE_EDITOR_ROLE - from google.cloud.storage.iam import STORAGE_VIEWER_ROLE - from google.api_core.iam import Policy - - BLOB_NAME = "blob-name" - PATH = "/b/name/o/%s" % (BLOB_NAME,) - ETAG = "DEADBEEF" - VERSION = 1 - OWNER1 = "user:phred@example.com" - OWNER2 = "group:cloud-logs@google.com" - EDITOR1 = "domain:google.com" - EDITOR2 = "user:phred@example.com" - VIEWER1 = "serviceAccount:1234-abcdef@service.example.com" - VIEWER2 = "user:phred@example.com" - RETURNED = { - "resourceId": PATH, - "etag": ETAG, - "version": VERSION, - "bindings": [ - {"role": STORAGE_OWNER_ROLE, "members": [OWNER1, OWNER2]}, - {"role": STORAGE_EDITOR_ROLE, "members": [EDITOR1, EDITOR2]}, - {"role": STORAGE_VIEWER_ROLE, "members": [VIEWER1, VIEWER2]}, - ], - } - after = ({"status": http_client.OK}, RETURNED) - EXPECTED = { - binding["role"]: set(binding["members"]) for binding in RETURNED["bindings"] - } - connection = _Connection(after) - client = _Client(connection) - bucket = _Bucket(client=client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - - policy = blob.get_iam_policy(timeout=42) - - self.assertIsInstance(policy, Policy) - self.assertEqual(policy.etag, RETURNED["etag"]) - self.assertEqual(policy.version, RETURNED["version"]) - self.assertEqual(dict(policy), EXPECTED) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "GET", - "path": "%s/iam" % (PATH,), - "query_params": {}, - "_target_object": None, - "timeout": 42, - }, - ) - - def test_get_iam_policy_w_requested_policy_version(self): - from google.cloud.storage.iam import STORAGE_OWNER_ROLE - - BLOB_NAME = "blob-name" - PATH = "/b/name/o/%s" % (BLOB_NAME,) - ETAG = "DEADBEEF" - VERSION = 1 - OWNER1 = "user:phred@example.com" - OWNER2 = "group:cloud-logs@google.com" - RETURNED = { - "resourceId": PATH, - "etag": ETAG, - "version": VERSION, - "bindings": [{"role": STORAGE_OWNER_ROLE, "members": [OWNER1, OWNER2]}], - } - after = ({"status": http_client.OK}, RETURNED) - connection = _Connection(after) - client = _Client(connection) - bucket = _Bucket(client=client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - - blob.get_iam_policy(requested_policy_version=3) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "GET", - "path": "%s/iam" % (PATH,), - "query_params": {"optionsRequestedPolicyVersion": 3}, - "_target_object": None, - "timeout": self._get_default_timeout(), - }, - ) - - def test_get_iam_policy_w_user_project(self): - from google.api_core.iam import Policy - - BLOB_NAME = "blob-name" - USER_PROJECT = "user-project-123" - PATH = "/b/name/o/%s" % (BLOB_NAME,) - ETAG = "DEADBEEF" - VERSION = 1 - RETURNED = { - "resourceId": PATH, - "etag": ETAG, - "version": VERSION, - "bindings": [], - } - after = ({"status": http_client.OK}, RETURNED) - EXPECTED = {} - connection = _Connection(after) - client = _Client(connection) - bucket = _Bucket(client=client, user_project=USER_PROJECT) - blob = self._make_one(BLOB_NAME, bucket=bucket) - - policy = blob.get_iam_policy() - - self.assertIsInstance(policy, Policy) - self.assertEqual(policy.etag, RETURNED["etag"]) - self.assertEqual(policy.version, RETURNED["version"]) - self.assertEqual(dict(policy), EXPECTED) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "GET", - "path": "%s/iam" % (PATH,), - "query_params": {"userProject": USER_PROJECT}, - "_target_object": None, - "timeout": self._get_default_timeout(), - }, - ) - - def test_set_iam_policy(self): - import operator - from google.cloud.storage.iam import STORAGE_OWNER_ROLE - from google.cloud.storage.iam import STORAGE_EDITOR_ROLE - from google.cloud.storage.iam import STORAGE_VIEWER_ROLE - from google.api_core.iam import Policy - - BLOB_NAME = "blob-name" - PATH = "/b/name/o/%s" % (BLOB_NAME,) - ETAG = "DEADBEEF" - VERSION = 1 - OWNER1 = "user:phred@example.com" - OWNER2 = "group:cloud-logs@google.com" - EDITOR1 = "domain:google.com" - EDITOR2 = "user:phred@example.com" - VIEWER1 = "serviceAccount:1234-abcdef@service.example.com" - VIEWER2 = "user:phred@example.com" - BINDINGS = [ - {"role": STORAGE_OWNER_ROLE, "members": [OWNER1, OWNER2]}, - {"role": STORAGE_EDITOR_ROLE, "members": [EDITOR1, EDITOR2]}, - {"role": STORAGE_VIEWER_ROLE, "members": [VIEWER1, VIEWER2]}, - ] - RETURNED = {"etag": ETAG, "version": VERSION, "bindings": BINDINGS} - after = ({"status": http_client.OK}, RETURNED) - policy = Policy() - for binding in BINDINGS: - policy[binding["role"]] = binding["members"] - - connection = _Connection(after) - client = _Client(connection) - bucket = _Bucket(client=client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - - returned = blob.set_iam_policy(policy, timeout=42) - - self.assertEqual(returned.etag, ETAG) - self.assertEqual(returned.version, VERSION) - self.assertEqual(dict(returned), dict(policy)) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "PUT") - self.assertEqual(kw[0]["path"], "%s/iam" % (PATH,)) - self.assertEqual(kw[0]["query_params"], {}) - self.assertEqual(kw[0]["timeout"], 42) - sent = kw[0]["data"] - self.assertEqual(sent["resourceId"], PATH) - self.assertEqual(len(sent["bindings"]), len(BINDINGS)) - key = operator.itemgetter("role") - for found, expected in zip( - sorted(sent["bindings"], key=key), sorted(BINDINGS, key=key) - ): - self.assertEqual(found["role"], expected["role"]) - self.assertEqual(sorted(found["members"]), sorted(expected["members"])) - - def test_set_iam_policy_w_user_project(self): - from google.api_core.iam import Policy - - BLOB_NAME = "blob-name" - USER_PROJECT = "user-project-123" - PATH = "/b/name/o/%s" % (BLOB_NAME,) - ETAG = "DEADBEEF" - VERSION = 1 - BINDINGS = [] - RETURNED = {"etag": ETAG, "version": VERSION, "bindings": BINDINGS} - after = ({"status": http_client.OK}, RETURNED) - policy = Policy() - - connection = _Connection(after) - client = _Client(connection) - bucket = _Bucket(client=client, user_project=USER_PROJECT) - blob = self._make_one(BLOB_NAME, bucket=bucket) - - returned = blob.set_iam_policy(policy) - - self.assertEqual(returned.etag, ETAG) - self.assertEqual(returned.version, VERSION) - self.assertEqual(dict(returned), dict(policy)) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "PUT") - self.assertEqual(kw[0]["path"], "%s/iam" % (PATH,)) - self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) - self.assertEqual(kw[0]["data"], {"resourceId": PATH}) - - def test_test_iam_permissions(self): - from google.cloud.storage.iam import STORAGE_OBJECTS_LIST - from google.cloud.storage.iam import STORAGE_BUCKETS_GET - from google.cloud.storage.iam import STORAGE_BUCKETS_UPDATE - - BLOB_NAME = "blob-name" - PATH = "/b/name/o/%s" % (BLOB_NAME,) - PERMISSIONS = [ - STORAGE_OBJECTS_LIST, - STORAGE_BUCKETS_GET, - STORAGE_BUCKETS_UPDATE, - ] - ALLOWED = PERMISSIONS[1:] - RETURNED = {"permissions": ALLOWED} - after = ({"status": http_client.OK}, RETURNED) - connection = _Connection(after) - client = _Client(connection) - bucket = _Bucket(client=client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - - allowed = blob.test_iam_permissions(PERMISSIONS, timeout=42) - - self.assertEqual(allowed, ALLOWED) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "GET") - self.assertEqual(kw[0]["path"], "%s/iam/testPermissions" % (PATH,)) - self.assertEqual(kw[0]["query_params"], {"permissions": PERMISSIONS}) - self.assertEqual(kw[0]["timeout"], 42) - - def test_test_iam_permissions_w_user_project(self): - from google.cloud.storage.iam import STORAGE_OBJECTS_LIST - from google.cloud.storage.iam import STORAGE_BUCKETS_GET - from google.cloud.storage.iam import STORAGE_BUCKETS_UPDATE - - BLOB_NAME = "blob-name" - USER_PROJECT = "user-project-123" - PATH = "/b/name/o/%s" % (BLOB_NAME,) - PERMISSIONS = [ - STORAGE_OBJECTS_LIST, - STORAGE_BUCKETS_GET, - STORAGE_BUCKETS_UPDATE, - ] - ALLOWED = PERMISSIONS[1:] - RETURNED = {"permissions": ALLOWED} - after = ({"status": http_client.OK}, RETURNED) - connection = _Connection(after) - client = _Client(connection) - bucket = _Bucket(client=client, user_project=USER_PROJECT) - blob = self._make_one(BLOB_NAME, bucket=bucket) - - allowed = blob.test_iam_permissions(PERMISSIONS) - - self.assertEqual(allowed, ALLOWED) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "GET") - self.assertEqual(kw[0]["path"], "%s/iam/testPermissions" % (PATH,)) - self.assertEqual( - kw[0]["query_params"], - {"permissions": PERMISSIONS, "userProject": USER_PROJECT}, - ) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - - def test_make_public(self): - from google.cloud.storage.acl import _ACLEntity - - BLOB_NAME = "blob-name" - permissive = [{"entity": "allUsers", "role": _ACLEntity.READER_ROLE}] - after = ({"status": http_client.OK}, {"acl": permissive}) - connection = _Connection(after) - client = _Client(connection) - bucket = _Bucket(client=client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - blob.acl.loaded = True - blob.make_public() - self.assertEqual(list(blob.acl), permissive) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "PATCH") - self.assertEqual(kw[0]["path"], "/b/name/o/%s" % BLOB_NAME) - self.assertEqual(kw[0]["data"], {"acl": permissive}) - self.assertEqual(kw[0]["query_params"], {"projection": "full"}) - - def test_make_private(self): - BLOB_NAME = "blob-name" - no_permissions = [] - after = ({"status": http_client.OK}, {"acl": no_permissions}) - connection = _Connection(after) - client = _Client(connection) - bucket = _Bucket(client=client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - blob.acl.loaded = True - blob.make_private() - self.assertEqual(list(blob.acl), no_permissions) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "PATCH") - self.assertEqual(kw[0]["path"], "/b/name/o/%s" % BLOB_NAME) - self.assertEqual(kw[0]["data"], {"acl": no_permissions}) - self.assertEqual(kw[0]["query_params"], {"projection": "full"}) - - def test_compose_wo_content_type_set(self): - SOURCE_1 = "source-1" - SOURCE_2 = "source-2" - DESTINATION = "destinaton" - RESOURCE = {} - after = ({"status": http_client.OK}, RESOURCE) - connection = _Connection(after) - client = _Client(connection) - bucket = _Bucket(client=client) - source_1 = self._make_one(SOURCE_1, bucket=bucket) - source_2 = self._make_one(SOURCE_2, bucket=bucket) - destination = self._make_one(DESTINATION, bucket=bucket) - # no destination.content_type set - - destination.compose(sources=[source_1, source_2]) - - self.assertIsNone(destination.content_type) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "POST", - "path": "/b/name/o/%s/compose" % DESTINATION, - "query_params": {}, - "data": { - "sourceObjects": [{"name": source_1.name}, {"name": source_2.name}], - "destination": {}, - }, - "_target_object": destination, - "timeout": self._get_default_timeout(), - }, - ) - - def test_compose_minimal_w_user_project(self): - SOURCE_1 = "source-1" - SOURCE_2 = "source-2" - DESTINATION = "destinaton" - RESOURCE = {"etag": "DEADBEEF"} - USER_PROJECT = "user-project-123" - after = ({"status": http_client.OK}, RESOURCE) - connection = _Connection(after) - client = _Client(connection) - bucket = _Bucket(client=client, user_project=USER_PROJECT) - source_1 = self._make_one(SOURCE_1, bucket=bucket) - source_2 = self._make_one(SOURCE_2, bucket=bucket) - destination = self._make_one(DESTINATION, bucket=bucket) - destination.content_type = "text/plain" - - destination.compose(sources=[source_1, source_2], timeout=42) - - self.assertEqual(destination.etag, "DEADBEEF") - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "POST", - "path": "/b/name/o/%s/compose" % DESTINATION, - "query_params": {"userProject": USER_PROJECT}, - "data": { - "sourceObjects": [{"name": source_1.name}, {"name": source_2.name}], - "destination": {"contentType": "text/plain"}, - }, - "_target_object": destination, - "timeout": 42, - }, - ) - - def test_compose_w_additional_property_changes(self): - SOURCE_1 = "source-1" - SOURCE_2 = "source-2" - DESTINATION = "destinaton" - RESOURCE = {"etag": "DEADBEEF"} - after = ({"status": http_client.OK}, RESOURCE) - connection = _Connection(after) - client = _Client(connection) - bucket = _Bucket(client=client) - source_1 = self._make_one(SOURCE_1, bucket=bucket) - source_2 = self._make_one(SOURCE_2, bucket=bucket) - destination = self._make_one(DESTINATION, bucket=bucket) - destination.content_type = "text/plain" - destination.content_language = "en-US" - destination.metadata = {"my-key": "my-value"} - - destination.compose(sources=[source_1, source_2]) - - self.assertEqual(destination.etag, "DEADBEEF") - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual( - kw[0], - { - "method": "POST", - "path": "/b/name/o/%s/compose" % DESTINATION, - "query_params": {}, - "data": { - "sourceObjects": [{"name": source_1.name}, {"name": source_2.name}], - "destination": { - "contentType": "text/plain", - "contentLanguage": "en-US", - "metadata": {"my-key": "my-value"}, - }, - }, - "_target_object": destination, - "timeout": self._get_default_timeout(), - }, - ) - - def test_rewrite_response_without_resource(self): - SOURCE_BLOB = "source" - DEST_BLOB = "dest" - DEST_BUCKET = "other-bucket" - TOKEN = "TOKEN" - RESPONSE = { - "totalBytesRewritten": 33, - "objectSize": 42, - "done": False, - "rewriteToken": TOKEN, - } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) - source_bucket = _Bucket(client=client) - source_blob = self._make_one(SOURCE_BLOB, bucket=source_bucket) - dest_bucket = _Bucket(client=client, name=DEST_BUCKET) - dest_blob = self._make_one(DEST_BLOB, bucket=dest_bucket) - - token, rewritten, size = dest_blob.rewrite(source_blob) - - self.assertEqual(token, TOKEN) - self.assertEqual(rewritten, 33) - self.assertEqual(size, 42) - - def test_rewrite_w_generations(self): - SOURCE_BLOB = "source" - SOURCE_GENERATION = 42 - DEST_BLOB = "dest" - DEST_BUCKET = "other-bucket" - DEST_GENERATION = 43 - TOKEN = "TOKEN" - RESPONSE = { - "totalBytesRewritten": 33, - "objectSize": 42, - "done": False, - "rewriteToken": TOKEN, - } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) - source_bucket = _Bucket(client=client) - source_blob = self._make_one( - SOURCE_BLOB, bucket=source_bucket, generation=SOURCE_GENERATION - ) - dest_bucket = _Bucket(client=client, name=DEST_BUCKET) - dest_blob = self._make_one( - DEST_BLOB, bucket=dest_bucket, generation=DEST_GENERATION - ) - - token, rewritten, size = dest_blob.rewrite(source_blob, timeout=42) - - self.assertEqual(token, TOKEN) - self.assertEqual(rewritten, 33) - self.assertEqual(size, 42) - - (kw,) = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual( - kw["path"], - "/b/%s/o/%s/rewriteTo/b/%s/o/%s" - % ( - (source_bucket.name, source_blob.name, dest_bucket.name, dest_blob.name) - ), - ) - self.assertEqual(kw["query_params"], {"sourceGeneration": SOURCE_GENERATION}) - self.assertEqual(kw["timeout"], 42) - - def test_rewrite_other_bucket_other_name_no_encryption_partial(self): - SOURCE_BLOB = "source" - DEST_BLOB = "dest" - DEST_BUCKET = "other-bucket" - TOKEN = "TOKEN" - RESPONSE = { - "totalBytesRewritten": 33, - "objectSize": 42, - "done": False, - "rewriteToken": TOKEN, - "resource": {"etag": "DEADBEEF"}, - } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) - source_bucket = _Bucket(client=client) - source_blob = self._make_one(SOURCE_BLOB, bucket=source_bucket) - dest_bucket = _Bucket(client=client, name=DEST_BUCKET) - dest_blob = self._make_one(DEST_BLOB, bucket=dest_bucket) - - token, rewritten, size = dest_blob.rewrite(source_blob) - - self.assertEqual(token, TOKEN) - self.assertEqual(rewritten, 33) - self.assertEqual(size, 42) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "POST") - PATH = "/b/name/o/%s/rewriteTo/b/%s/o/%s" % ( - SOURCE_BLOB, - DEST_BUCKET, - DEST_BLOB, - ) - self.assertEqual(kw[0]["path"], PATH) - self.assertEqual(kw[0]["query_params"], {}) - SENT = {} - self.assertEqual(kw[0]["data"], SENT) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - - headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} - self.assertNotIn("X-Goog-Copy-Source-Encryption-Algorithm", headers) - self.assertNotIn("X-Goog-Copy-Source-Encryption-Key", headers) - self.assertNotIn("X-Goog-Copy-Source-Encryption-Key-Sha256", headers) - self.assertNotIn("X-Goog-Encryption-Algorithm", headers) - self.assertNotIn("X-Goog-Encryption-Key", headers) - self.assertNotIn("X-Goog-Encryption-Key-Sha256", headers) - - def test_rewrite_same_name_no_old_key_new_key_done_w_user_project(self): - KEY = b"01234567890123456789012345678901" # 32 bytes - KEY_B64 = base64.b64encode(KEY).rstrip().decode("ascii") - KEY_HASH = hashlib.sha256(KEY).digest() - KEY_HASH_B64 = base64.b64encode(KEY_HASH).rstrip().decode("ascii") - BLOB_NAME = "blob" - USER_PROJECT = "user-project-123" - RESPONSE = { - "totalBytesRewritten": 42, - "objectSize": 42, - "done": True, - "resource": {"etag": "DEADBEEF"}, - } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) - bucket = _Bucket(client=client, user_project=USER_PROJECT) - plain = self._make_one(BLOB_NAME, bucket=bucket) - encrypted = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=KEY) - - token, rewritten, size = encrypted.rewrite(plain) - - self.assertIsNone(token) - self.assertEqual(rewritten, 42) - self.assertEqual(size, 42) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "POST") - PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) - self.assertEqual(kw[0]["path"], PATH) - self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) - SENT = {} - self.assertEqual(kw[0]["data"], SENT) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - - headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} - self.assertNotIn("X-Goog-Copy-Source-Encryption-Algorithm", headers) - self.assertNotIn("X-Goog-Copy-Source-Encryption-Key", headers) - self.assertNotIn("X-Goog-Copy-Source-Encryption-Key-Sha256", headers) - self.assertEqual(headers["X-Goog-Encryption-Algorithm"], "AES256") - self.assertEqual(headers["X-Goog-Encryption-Key"], KEY_B64) - self.assertEqual(headers["X-Goog-Encryption-Key-Sha256"], KEY_HASH_B64) - - def test_rewrite_same_name_no_key_new_key_w_token(self): - SOURCE_KEY = b"01234567890123456789012345678901" # 32 bytes - SOURCE_KEY_B64 = base64.b64encode(SOURCE_KEY).rstrip().decode("ascii") - SOURCE_KEY_HASH = hashlib.sha256(SOURCE_KEY).digest() - SOURCE_KEY_HASH_B64 = base64.b64encode(SOURCE_KEY_HASH).rstrip().decode("ascii") - DEST_KEY = b"90123456789012345678901234567890" # 32 bytes - DEST_KEY_B64 = base64.b64encode(DEST_KEY).rstrip().decode("ascii") - DEST_KEY_HASH = hashlib.sha256(DEST_KEY).digest() - DEST_KEY_HASH_B64 = base64.b64encode(DEST_KEY_HASH).rstrip().decode("ascii") - BLOB_NAME = "blob" - TOKEN = "TOKEN" - RESPONSE = { - "totalBytesRewritten": 42, - "objectSize": 42, - "done": True, - "resource": {"etag": "DEADBEEF"}, - } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) - bucket = _Bucket(client=client) - source = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=SOURCE_KEY) - dest = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=DEST_KEY) - - token, rewritten, size = dest.rewrite(source, token=TOKEN) - - self.assertIsNone(token) - self.assertEqual(rewritten, 42) - self.assertEqual(size, 42) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "POST") - PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) - self.assertEqual(kw[0]["path"], PATH) - self.assertEqual(kw[0]["query_params"], {"rewriteToken": TOKEN}) - SENT = {} - self.assertEqual(kw[0]["data"], SENT) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - - headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} - self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Algorithm"], "AES256") - self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Key"], SOURCE_KEY_B64) - self.assertEqual( - headers["X-Goog-Copy-Source-Encryption-Key-Sha256"], SOURCE_KEY_HASH_B64 - ) - self.assertEqual(headers["X-Goog-Encryption-Algorithm"], "AES256") - self.assertEqual(headers["X-Goog-Encryption-Key"], DEST_KEY_B64) - self.assertEqual(headers["X-Goog-Encryption-Key-Sha256"], DEST_KEY_HASH_B64) - - def test_rewrite_same_name_w_old_key_new_kms_key(self): - SOURCE_KEY = b"01234567890123456789012345678901" # 32 bytes - SOURCE_KEY_B64 = base64.b64encode(SOURCE_KEY).rstrip().decode("ascii") - SOURCE_KEY_HASH = hashlib.sha256(SOURCE_KEY).digest() - SOURCE_KEY_HASH_B64 = base64.b64encode(SOURCE_KEY_HASH).rstrip().decode("ascii") - DEST_KMS_RESOURCE = ( - "projects/test-project-123/" - "locations/us/" - "keyRings/test-ring/" - "cryptoKeys/test-key" - ) - BLOB_NAME = "blob" - RESPONSE = { - "totalBytesRewritten": 42, - "objectSize": 42, - "done": True, - "resource": {"etag": "DEADBEEF"}, - } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) - bucket = _Bucket(client=client) - source = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=SOURCE_KEY) - dest = self._make_one(BLOB_NAME, bucket=bucket, kms_key_name=DEST_KMS_RESOURCE) - - token, rewritten, size = dest.rewrite(source) - - self.assertIsNone(token) - self.assertEqual(rewritten, 42) - self.assertEqual(size, 42) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "POST") - PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) - self.assertEqual(kw[0]["path"], PATH) - self.assertEqual( - kw[0]["query_params"], {"destinationKmsKeyName": DEST_KMS_RESOURCE} - ) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - SENT = {"kmsKeyName": DEST_KMS_RESOURCE} - self.assertEqual(kw[0]["data"], SENT) - - headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} - self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Algorithm"], "AES256") - self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Key"], SOURCE_KEY_B64) - self.assertEqual( - headers["X-Goog-Copy-Source-Encryption-Key-Sha256"], SOURCE_KEY_HASH_B64 - ) - - def test_update_storage_class_invalid(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - with self.assertRaises(ValueError): - blob.update_storage_class(u"BOGUS") - - def test_update_storage_class_large_file(self): - BLOB_NAME = "blob-name" - STORAGE_CLASS = u"NEARLINE" - TOKEN = "TOKEN" - INCOMPLETE_RESPONSE = { - "totalBytesRewritten": 42, - "objectSize": 84, - "done": False, - "rewriteToken": TOKEN, - "resource": {"storageClass": STORAGE_CLASS}, - } - COMPLETE_RESPONSE = { - "totalBytesRewritten": 84, - "objectSize": 84, - "done": True, - "resource": {"storageClass": STORAGE_CLASS}, - } - response_1 = ({"status": http_client.OK}, INCOMPLETE_RESPONSE) - response_2 = ({"status": http_client.OK}, COMPLETE_RESPONSE) - connection = _Connection(response_1, response_2) - client = _Client(connection) - bucket = _Bucket(client=client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - - blob.update_storage_class("NEARLINE") - - self.assertEqual(blob.storage_class, "NEARLINE") - - def test_update_storage_class_wo_encryption_key(self): - BLOB_NAME = "blob-name" - STORAGE_CLASS = u"NEARLINE" - RESPONSE = { - "totalBytesRewritten": 42, - "objectSize": 42, - "done": True, - "resource": {"storageClass": STORAGE_CLASS}, - } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) - bucket = _Bucket(client=client) - blob = self._make_one(BLOB_NAME, bucket=bucket) - - blob.update_storage_class("NEARLINE") - - self.assertEqual(blob.storage_class, "NEARLINE") - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "POST") - PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) - self.assertEqual(kw[0]["path"], PATH) - self.assertEqual(kw[0]["query_params"], {}) - SENT = {"storageClass": STORAGE_CLASS} - self.assertEqual(kw[0]["data"], SENT) - - headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} - # Blob has no key, and therefore the relevant headers are not sent. - self.assertNotIn("X-Goog-Copy-Source-Encryption-Algorithm", headers) - self.assertNotIn("X-Goog-Copy-Source-Encryption-Key", headers) - self.assertNotIn("X-Goog-Copy-Source-Encryption-Key-Sha256", headers) - self.assertNotIn("X-Goog-Encryption-Algorithm", headers) - self.assertNotIn("X-Goog-Encryption-Key", headers) - self.assertNotIn("X-Goog-Encryption-Key-Sha256", headers) - - def test_update_storage_class_w_encryption_key_w_user_project(self): - BLOB_NAME = "blob-name" - BLOB_KEY = b"01234567890123456789012345678901" # 32 bytes - BLOB_KEY_B64 = base64.b64encode(BLOB_KEY).rstrip().decode("ascii") - BLOB_KEY_HASH = hashlib.sha256(BLOB_KEY).digest() - BLOB_KEY_HASH_B64 = base64.b64encode(BLOB_KEY_HASH).rstrip().decode("ascii") - STORAGE_CLASS = u"NEARLINE" - USER_PROJECT = "user-project-123" - RESPONSE = { - "totalBytesRewritten": 42, - "objectSize": 42, - "done": True, - "resource": {"storageClass": STORAGE_CLASS}, - } - response = ({"status": http_client.OK}, RESPONSE) - connection = _Connection(response) - client = _Client(connection) - bucket = _Bucket(client=client, user_project=USER_PROJECT) - blob = self._make_one(BLOB_NAME, bucket=bucket, encryption_key=BLOB_KEY) - - blob.update_storage_class("NEARLINE") - - self.assertEqual(blob.storage_class, "NEARLINE") - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "POST") - PATH = "/b/name/o/%s/rewriteTo/b/name/o/%s" % (BLOB_NAME, BLOB_NAME) - self.assertEqual(kw[0]["path"], PATH) - self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) - SENT = {"storageClass": STORAGE_CLASS} - self.assertEqual(kw[0]["data"], SENT) - - headers = {key.title(): str(value) for key, value in kw[0]["headers"].items()} - # Blob has key, and therefore the relevant headers are sent. - self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Algorithm"], "AES256") - self.assertEqual(headers["X-Goog-Copy-Source-Encryption-Key"], BLOB_KEY_B64) - self.assertEqual( - headers["X-Goog-Copy-Source-Encryption-Key-Sha256"], BLOB_KEY_HASH_B64 - ) - self.assertEqual(headers["X-Goog-Encryption-Algorithm"], "AES256") - self.assertEqual(headers["X-Goog-Encryption-Key"], BLOB_KEY_B64) - self.assertEqual(headers["X-Goog-Encryption-Key-Sha256"], BLOB_KEY_HASH_B64) - - def test_cache_control_getter(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - CACHE_CONTROL = "no-cache" - properties = {"cacheControl": CACHE_CONTROL} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.cache_control, CACHE_CONTROL) - - def test_cache_control_setter(self): - BLOB_NAME = "blob-name" - CACHE_CONTROL = "no-cache" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertIsNone(blob.cache_control) - blob.cache_control = CACHE_CONTROL - self.assertEqual(blob.cache_control, CACHE_CONTROL) - - def test_component_count(self): - BUCKET = object() - COMPONENT_COUNT = 42 - blob = self._make_one( - "blob-name", bucket=BUCKET, properties={"componentCount": COMPONENT_COUNT} - ) - self.assertEqual(blob.component_count, COMPONENT_COUNT) - - def test_component_count_unset(self): - BUCKET = object() - blob = self._make_one("blob-name", bucket=BUCKET) - self.assertIsNone(blob.component_count) - - def test_component_count_string_val(self): - BUCKET = object() - COMPONENT_COUNT = 42 - blob = self._make_one( - "blob-name", - bucket=BUCKET, - properties={"componentCount": str(COMPONENT_COUNT)}, - ) - self.assertEqual(blob.component_count, COMPONENT_COUNT) - - def test_content_disposition_getter(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - CONTENT_DISPOSITION = "Attachment; filename=example.jpg" - properties = {"contentDisposition": CONTENT_DISPOSITION} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.content_disposition, CONTENT_DISPOSITION) - - def test_content_disposition_setter(self): - BLOB_NAME = "blob-name" - CONTENT_DISPOSITION = "Attachment; filename=example.jpg" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertIsNone(blob.content_disposition) - blob.content_disposition = CONTENT_DISPOSITION - self.assertEqual(blob.content_disposition, CONTENT_DISPOSITION) - - def test_content_encoding_getter(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - CONTENT_ENCODING = "gzip" - properties = {"contentEncoding": CONTENT_ENCODING} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.content_encoding, CONTENT_ENCODING) - - def test_content_encoding_setter(self): - BLOB_NAME = "blob-name" - CONTENT_ENCODING = "gzip" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertIsNone(blob.content_encoding) - blob.content_encoding = CONTENT_ENCODING - self.assertEqual(blob.content_encoding, CONTENT_ENCODING) - - def test_content_language_getter(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - CONTENT_LANGUAGE = "pt-BR" - properties = {"contentLanguage": CONTENT_LANGUAGE} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.content_language, CONTENT_LANGUAGE) - - def test_content_language_setter(self): - BLOB_NAME = "blob-name" - CONTENT_LANGUAGE = "pt-BR" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertIsNone(blob.content_language) - blob.content_language = CONTENT_LANGUAGE - self.assertEqual(blob.content_language, CONTENT_LANGUAGE) - - def test_content_type_getter(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - CONTENT_TYPE = "image/jpeg" - properties = {"contentType": CONTENT_TYPE} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.content_type, CONTENT_TYPE) - - def test_content_type_setter(self): - BLOB_NAME = "blob-name" - CONTENT_TYPE = "image/jpeg" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertIsNone(blob.content_type) - blob.content_type = CONTENT_TYPE - self.assertEqual(blob.content_type, CONTENT_TYPE) - - def test_crc32c_getter(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - CRC32C = "DEADBEEF" - properties = {"crc32c": CRC32C} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.crc32c, CRC32C) - - def test_crc32c_setter(self): - BLOB_NAME = "blob-name" - CRC32C = "DEADBEEF" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertIsNone(blob.crc32c) - blob.crc32c = CRC32C - self.assertEqual(blob.crc32c, CRC32C) - - def test_etag(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - ETAG = "ETAG" - properties = {"etag": ETAG} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.etag, ETAG) - - def test_event_based_hold_getter_missing(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - properties = {} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertIsNone(blob.event_based_hold) - - def test_event_based_hold_getter_false(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - properties = {"eventBasedHold": False} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertFalse(blob.event_based_hold) - - def test_event_based_hold_getter_true(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - properties = {"eventBasedHold": True} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertTrue(blob.event_based_hold) - - def test_event_based_hold_setter(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertIsNone(blob.event_based_hold) - blob.event_based_hold = True - self.assertEqual(blob.event_based_hold, True) - - def test_generation(self): - BUCKET = object() - GENERATION = 42 - blob = self._make_one( - "blob-name", bucket=BUCKET, properties={"generation": GENERATION} - ) - self.assertEqual(blob.generation, GENERATION) - - def test_generation_unset(self): - BUCKET = object() - blob = self._make_one("blob-name", bucket=BUCKET) - self.assertIsNone(blob.generation) - - def test_generation_string_val(self): - BUCKET = object() - GENERATION = 42 - blob = self._make_one( - "blob-name", bucket=BUCKET, properties={"generation": str(GENERATION)} - ) - self.assertEqual(blob.generation, GENERATION) - - def test_id(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - ID = "ID" - properties = {"id": ID} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.id, ID) - - def test_md5_hash_getter(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - MD5_HASH = "DEADBEEF" - properties = {"md5Hash": MD5_HASH} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.md5_hash, MD5_HASH) - - def test_md5_hash_setter(self): - BLOB_NAME = "blob-name" - MD5_HASH = "DEADBEEF" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertIsNone(blob.md5_hash) - blob.md5_hash = MD5_HASH - self.assertEqual(blob.md5_hash, MD5_HASH) - - def test_media_link(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - MEDIA_LINK = "http://example.com/media/" - properties = {"mediaLink": MEDIA_LINK} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.media_link, MEDIA_LINK) - - def test_metadata_getter(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - METADATA = {"foo": "Foo"} - properties = {"metadata": METADATA} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.metadata, METADATA) - - def test_metadata_setter(self): - BLOB_NAME = "blob-name" - METADATA = {"foo": "Foo"} - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertIsNone(blob.metadata) - blob.metadata = METADATA - self.assertEqual(blob.metadata, METADATA) - - def test_metadata_setter_w_nan(self): - BLOB_NAME = "blob-name" - METADATA = {"foo": float("nan")} - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertIsNone(blob.metadata) - blob.metadata = METADATA - value = blob.metadata["foo"] - self.assertIsInstance(value, str) - - def test_metageneration(self): - BUCKET = object() - METAGENERATION = 42 - blob = self._make_one( - "blob-name", bucket=BUCKET, properties={"metageneration": METAGENERATION} - ) - self.assertEqual(blob.metageneration, METAGENERATION) - - def test_metageneration_unset(self): - BUCKET = object() - blob = self._make_one("blob-name", bucket=BUCKET) - self.assertIsNone(blob.metageneration) - - def test_metageneration_string_val(self): - BUCKET = object() - METAGENERATION = 42 - blob = self._make_one( - "blob-name", - bucket=BUCKET, - properties={"metageneration": str(METAGENERATION)}, - ) - self.assertEqual(blob.metageneration, METAGENERATION) - - def test_owner(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - OWNER = {"entity": "project-owner-12345", "entityId": "23456"} - properties = {"owner": OWNER} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - owner = blob.owner - self.assertEqual(owner["entity"], "project-owner-12345") - self.assertEqual(owner["entityId"], "23456") - - def test_retention_expiration_time(self): - from google.cloud._helpers import _RFC3339_MICROS - from google.cloud._helpers import UTC - - BLOB_NAME = "blob-name" - bucket = _Bucket() - TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=UTC) - TIME_CREATED = TIMESTAMP.strftime(_RFC3339_MICROS) - properties = {"retentionExpirationTime": TIME_CREATED} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.retention_expiration_time, TIMESTAMP) - - def test_retention_expiration_time_unset(self): - BUCKET = object() - blob = self._make_one("blob-name", bucket=BUCKET) - self.assertIsNone(blob.retention_expiration_time) - - def test_self_link(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - SELF_LINK = "http://example.com/self/" - properties = {"selfLink": SELF_LINK} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.self_link, SELF_LINK) - - def test_size(self): - BUCKET = object() - SIZE = 42 - blob = self._make_one("blob-name", bucket=BUCKET, properties={"size": SIZE}) - self.assertEqual(blob.size, SIZE) - - def test_size_unset(self): - BUCKET = object() - blob = self._make_one("blob-name", bucket=BUCKET) - self.assertIsNone(blob.size) - - def test_size_string_val(self): - BUCKET = object() - SIZE = 42 - blob = self._make_one( - "blob-name", bucket=BUCKET, properties={"size": str(SIZE)} - ) - self.assertEqual(blob.size, SIZE) - - def test_storage_class_getter(self): - blob_name = "blob-name" - bucket = _Bucket() - storage_class = "COLDLINE" - properties = {"storageClass": storage_class} - blob = self._make_one(blob_name, bucket=bucket, properties=properties) - self.assertEqual(blob.storage_class, storage_class) - - def test_storage_class_setter(self): - blob_name = "blob-name" - bucket = _Bucket() - storage_class = "COLDLINE" - blob = self._make_one(blob_name, bucket=bucket) - self.assertIsNone(blob.storage_class) - blob.storage_class = storage_class - self.assertEqual(blob.storage_class, storage_class) - self.assertEqual(blob._properties, {"storageClass": storage_class}) - - def test_temporary_hold_getter_missing(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - properties = {} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertIsNone(blob.temporary_hold) - - def test_temporary_hold_getter_false(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - properties = {"temporaryHold": False} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertFalse(blob.temporary_hold) - - def test_temporary_hold_getter_true(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - properties = {"temporaryHold": True} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertTrue(blob.temporary_hold) - - def test_temporary_hold_setter(self): - BLOB_NAME = "blob-name" - bucket = _Bucket() - blob = self._make_one(BLOB_NAME, bucket=bucket) - self.assertIsNone(blob.temporary_hold) - blob.temporary_hold = True - self.assertEqual(blob.temporary_hold, True) - - def test_time_deleted(self): - from google.cloud._helpers import _RFC3339_MICROS - from google.cloud._helpers import UTC - - BLOB_NAME = "blob-name" - bucket = _Bucket() - TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=UTC) - TIME_DELETED = TIMESTAMP.strftime(_RFC3339_MICROS) - properties = {"timeDeleted": TIME_DELETED} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.time_deleted, TIMESTAMP) - - def test_time_deleted_unset(self): - BUCKET = object() - blob = self._make_one("blob-name", bucket=BUCKET) - self.assertIsNone(blob.time_deleted) - - def test_time_created(self): - from google.cloud._helpers import _RFC3339_MICROS - from google.cloud._helpers import UTC - - BLOB_NAME = "blob-name" - bucket = _Bucket() - TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=UTC) - TIME_CREATED = TIMESTAMP.strftime(_RFC3339_MICROS) - properties = {"timeCreated": TIME_CREATED} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.time_created, TIMESTAMP) - - def test_time_created_unset(self): - BUCKET = object() - blob = self._make_one("blob-name", bucket=BUCKET) - self.assertIsNone(blob.time_created) - - def test_updated(self): - from google.cloud._helpers import _RFC3339_MICROS - from google.cloud._helpers import UTC - - BLOB_NAME = "blob-name" - bucket = _Bucket() - TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=UTC) - UPDATED = TIMESTAMP.strftime(_RFC3339_MICROS) - properties = {"updated": UPDATED} - blob = self._make_one(BLOB_NAME, bucket=bucket, properties=properties) - self.assertEqual(blob.updated, TIMESTAMP) - - def test_updated_unset(self): - BUCKET = object() - blob = self._make_one("blob-name", bucket=BUCKET) - self.assertIsNone(blob.updated) - - def test_from_string_w_valid_uri(self): - from google.cloud.storage.blob import Blob - - connection = _Connection() - client = _Client(connection) - uri = "gs://BUCKET_NAME/b" - blob = Blob.from_string(uri, client) - - self.assertIsInstance(blob, Blob) - self.assertIs(blob.client, client) - self.assertEqual(blob.name, "b") - self.assertEqual(blob.bucket.name, "BUCKET_NAME") - - def test_from_string_w_invalid_uri(self): - from google.cloud.storage.blob import Blob - - connection = _Connection() - client = _Client(connection) - - with pytest.raises(ValueError, match="URI scheme must be gs"): - Blob.from_string("http://bucket_name/b", client) - - def test_from_string_w_domain_name_bucket(self): - from google.cloud.storage.blob import Blob - - connection = _Connection() - client = _Client(connection) - uri = "gs://buckets.example.com/b" - blob = Blob.from_string(uri, client) - - self.assertIsInstance(blob, Blob) - self.assertIs(blob.client, client) - self.assertEqual(blob.name, "b") - self.assertEqual(blob.bucket.name, "buckets.example.com") - - -class Test__quote(unittest.TestCase): - @staticmethod - def _call_fut(*args, **kw): - from google.cloud.storage.blob import _quote - - return _quote(*args, **kw) - - def test_bytes(self): - quoted = self._call_fut(b"\xDE\xAD\xBE\xEF") - self.assertEqual(quoted, "%DE%AD%BE%EF") - - def test_unicode(self): - helicopter = u"\U0001f681" - quoted = self._call_fut(helicopter) - self.assertEqual(quoted, "%F0%9F%9A%81") - - def test_bad_type(self): - with self.assertRaises(TypeError): - self._call_fut(None) - - def test_w_slash_default(self): - with_slash = "foo/bar/baz" - quoted = self._call_fut(with_slash) - self.assertEqual(quoted, "foo%2Fbar%2Fbaz") - - def test_w_slash_w_safe(self): - with_slash = "foo/bar/baz" - quoted_safe = self._call_fut(with_slash, safe=b"/") - self.assertEqual(quoted_safe, with_slash) - - def test_w_tilde(self): - with_tilde = "bam~qux" - quoted = self._call_fut(with_tilde, safe=b"~") - self.assertEqual(quoted, with_tilde) - - -class Test__maybe_rewind(unittest.TestCase): - @staticmethod - def _call_fut(*args, **kwargs): - from google.cloud.storage.blob import _maybe_rewind - - return _maybe_rewind(*args, **kwargs) - - def test_default(self): - stream = mock.Mock(spec=[u"seek"]) - ret_val = self._call_fut(stream) - self.assertIsNone(ret_val) - - stream.seek.assert_not_called() - - def test_do_not_rewind(self): - stream = mock.Mock(spec=[u"seek"]) - ret_val = self._call_fut(stream, rewind=False) - self.assertIsNone(ret_val) - - stream.seek.assert_not_called() - - def test_do_rewind(self): - stream = mock.Mock(spec=[u"seek"]) - ret_val = self._call_fut(stream, rewind=True) - self.assertIsNone(ret_val) - - stream.seek.assert_called_once_with(0, os.SEEK_SET) - - -class Test__raise_from_invalid_response(unittest.TestCase): - @staticmethod - def _call_fut(error): - from google.cloud.storage.blob import _raise_from_invalid_response - - return _raise_from_invalid_response(error) - - def _helper(self, message, code=http_client.BAD_REQUEST, args=()): - import requests - - from google.resumable_media import InvalidResponse - from google.api_core import exceptions - - response = requests.Response() - response.request = requests.Request("GET", "http://example.com").prepare() - response.status_code = code - error = InvalidResponse(response, message, *args) - - with self.assertRaises(exceptions.GoogleAPICallError) as exc_info: - self._call_fut(error) - - return exc_info - - def test_default(self): - message = "Failure" - exc_info = self._helper(message) - expected = "GET http://example.com/: {}".format(message) - self.assertEqual(exc_info.exception.message, expected) - self.assertEqual(exc_info.exception.errors, []) - - def test_w_206_and_args(self): - message = "Failure" - args = ("one", "two") - exc_info = self._helper(message, code=http_client.PARTIAL_CONTENT, args=args) - expected = "GET http://example.com/: {}".format((message,) + args) - self.assertEqual(exc_info.exception.message, expected) - self.assertEqual(exc_info.exception.errors, []) - - -class Test__add_query_parameters(unittest.TestCase): - @staticmethod - def _call_fut(*args, **kwargs): - from google.cloud.storage.blob import _add_query_parameters - - return _add_query_parameters(*args, **kwargs) - - def test_w_empty_list(self): - BASE_URL = "https://test.example.com/base" - self.assertEqual(self._call_fut(BASE_URL, []), BASE_URL) - - def test_wo_existing_qs(self): - BASE_URL = "https://test.example.com/base" - NV_LIST = [("one", "One"), ("two", "Two")] - expected = "&".join(["{}={}".format(name, value) for name, value in NV_LIST]) - self.assertEqual( - self._call_fut(BASE_URL, NV_LIST), "{}?{}".format(BASE_URL, expected) - ) - - def test_w_existing_qs(self): - BASE_URL = "https://test.example.com/base?one=Three" - NV_LIST = [("one", "One"), ("two", "Two")] - expected = "&".join(["{}={}".format(name, value) for name, value in NV_LIST]) - self.assertEqual( - self._call_fut(BASE_URL, NV_LIST), "{}&{}".format(BASE_URL, expected) - ) - - -class _Connection(object): - - API_BASE_URL = "http://example.com" - USER_AGENT = "testing 1.2.3" - credentials = object() - - def __init__(self, *responses): - self._responses = responses[:] - self._requested = [] - self._signed = [] - - def _respond(self, **kw): - self._requested.append(kw) - response, self._responses = self._responses[0], self._responses[1:] - return response - - def api_request(self, **kw): - from google.cloud.exceptions import NotFound - - info, content = self._respond(**kw) - if info.get("status") == http_client.NOT_FOUND: - raise NotFound(info) - return content - - -class _Bucket(object): - def __init__(self, client=None, name="name", user_project=None): - if client is None: - connection = _Connection() - client = _Client(connection) - self.client = client - self._blobs = {} - self._copied = [] - self._deleted = [] - self.name = name - self.path = "/b/" + name - self.user_project = user_project - - def delete_blob(self, blob_name, client=None, generation=None, timeout=None): - del self._blobs[blob_name] - self._deleted.append((blob_name, client, generation, timeout)) - - -class _Client(object): - def __init__(self, connection): - self._base_connection = connection - - @property - def _connection(self): - return self._base_connection - - @property - def _credentials(self): - return self._base_connection.credentials diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py deleted file mode 100644 index 365e1f0e1..000000000 --- a/tests/unit/test_bucket.py +++ /dev/null @@ -1,3038 +0,0 @@ -# Copyright 2014 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import unittest - -import mock -import pytest - - -def _make_connection(*responses): - import google.cloud.storage._http - - mock_connection = mock.create_autospec(google.cloud.storage._http.Connection) - mock_connection.user_agent = "testing 1.2.3" - mock_connection.api_request.side_effect = list(responses) - return mock_connection - - -def _create_signing_credentials(): - import google.auth.credentials - - class _SigningCredentials( - google.auth.credentials.Credentials, google.auth.credentials.Signing - ): - pass - - credentials = mock.Mock(spec=_SigningCredentials) - - return credentials - - -class Test_LifecycleRuleConditions(unittest.TestCase): - @staticmethod - def _get_target_class(): - from google.cloud.storage.bucket import LifecycleRuleConditions - - return LifecycleRuleConditions - - def _make_one(self, **kw): - return self._get_target_class()(**kw) - - def test_ctor_wo_conditions(self): - with self.assertRaises(ValueError): - self._make_one() - - def test_ctor_w_age_and_matches_storage_class(self): - conditions = self._make_one(age=10, matches_storage_class=["COLDLINE"]) - expected = {"age": 10, "matchesStorageClass": ["COLDLINE"]} - self.assertEqual(dict(conditions), expected) - self.assertEqual(conditions.age, 10) - self.assertIsNone(conditions.created_before) - self.assertIsNone(conditions.is_live) - self.assertEqual(conditions.matches_storage_class, ["COLDLINE"]) - self.assertIsNone(conditions.number_of_newer_versions) - - def test_ctor_w_created_before_and_is_live(self): - import datetime - - before = datetime.date(2018, 8, 1) - conditions = self._make_one(created_before=before, is_live=False) - expected = {"createdBefore": "2018-08-01", "isLive": False} - self.assertEqual(dict(conditions), expected) - self.assertIsNone(conditions.age) - self.assertEqual(conditions.created_before, before) - self.assertEqual(conditions.is_live, False) - self.assertIsNone(conditions.matches_storage_class) - self.assertIsNone(conditions.number_of_newer_versions) - - def test_ctor_w_number_of_newer_versions(self): - conditions = self._make_one(number_of_newer_versions=3) - expected = {"numNewerVersions": 3} - self.assertEqual(dict(conditions), expected) - self.assertIsNone(conditions.age) - self.assertIsNone(conditions.created_before) - self.assertIsNone(conditions.is_live) - self.assertIsNone(conditions.matches_storage_class) - self.assertEqual(conditions.number_of_newer_versions, 3) - - def test_from_api_repr(self): - import datetime - - before = datetime.date(2018, 8, 1) - klass = self._get_target_class() - resource = { - "age": 10, - "createdBefore": "2018-08-01", - "isLive": True, - "matchesStorageClass": ["COLDLINE"], - "numNewerVersions": 3, - } - conditions = klass.from_api_repr(resource) - self.assertEqual(conditions.age, 10) - self.assertEqual(conditions.created_before, before) - self.assertEqual(conditions.is_live, True) - self.assertEqual(conditions.matches_storage_class, ["COLDLINE"]) - self.assertEqual(conditions.number_of_newer_versions, 3) - - -class Test_LifecycleRuleDelete(unittest.TestCase): - @staticmethod - def _get_target_class(): - from google.cloud.storage.bucket import LifecycleRuleDelete - - return LifecycleRuleDelete - - def _make_one(self, **kw): - return self._get_target_class()(**kw) - - def test_ctor_wo_conditions(self): - with self.assertRaises(ValueError): - self._make_one() - - def test_ctor_w_condition(self): - rule = self._make_one(age=10, matches_storage_class=["COLDLINE"]) - expected = { - "action": {"type": "Delete"}, - "condition": {"age": 10, "matchesStorageClass": ["COLDLINE"]}, - } - self.assertEqual(dict(rule), expected) - - def test_from_api_repr(self): - klass = self._get_target_class() - conditions = { - "age": 10, - "createdBefore": "2018-08-01", - "isLive": True, - "matchesStorageClass": ["COLDLINE"], - "numNewerVersions": 3, - } - resource = {"action": {"type": "Delete"}, "condition": conditions} - rule = klass.from_api_repr(resource) - self.assertEqual(dict(rule), resource) - - -class Test_LifecycleRuleSetStorageClass(unittest.TestCase): - @staticmethod - def _get_target_class(): - from google.cloud.storage.bucket import LifecycleRuleSetStorageClass - - return LifecycleRuleSetStorageClass - - def _make_one(self, **kw): - return self._get_target_class()(**kw) - - def test_ctor_wo_conditions(self): - with self.assertRaises(ValueError): - self._make_one(storage_class="COLDLINE") - - def test_ctor_w_condition(self): - rule = self._make_one( - storage_class="COLDLINE", age=10, matches_storage_class=["NEARLINE"] - ) - expected = { - "action": {"type": "SetStorageClass", "storageClass": "COLDLINE"}, - "condition": {"age": 10, "matchesStorageClass": ["NEARLINE"]}, - } - self.assertEqual(dict(rule), expected) - - def test_from_api_repr(self): - klass = self._get_target_class() - conditions = { - "age": 10, - "createdBefore": "2018-08-01", - "isLive": True, - "matchesStorageClass": ["NEARLINE"], - "numNewerVersions": 3, - } - resource = { - "action": {"type": "SetStorageClass", "storageClass": "COLDLINE"}, - "condition": conditions, - } - rule = klass.from_api_repr(resource) - self.assertEqual(dict(rule), resource) - - -class Test_IAMConfiguration(unittest.TestCase): - @staticmethod - def _get_target_class(): - from google.cloud.storage.bucket import IAMConfiguration - - return IAMConfiguration - - def _make_one(self, bucket, **kw): - return self._get_target_class()(bucket, **kw) - - @staticmethod - def _make_bucket(): - from google.cloud.storage.bucket import Bucket - - return mock.create_autospec(Bucket, instance=True) - - def test_ctor_defaults(self): - bucket = self._make_bucket() - - config = self._make_one(bucket) - - self.assertIs(config.bucket, bucket) - self.assertFalse(config.uniform_bucket_level_access_enabled) - self.assertIsNone(config.uniform_bucket_level_access_locked_time) - self.assertFalse(config.bucket_policy_only_enabled) - self.assertIsNone(config.bucket_policy_only_locked_time) - - def test_ctor_explicit_ubla(self): - import datetime - import pytz - - bucket = self._make_bucket() - now = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) - - config = self._make_one( - bucket, - uniform_bucket_level_access_enabled=True, - uniform_bucket_level_access_locked_time=now, - ) - - self.assertIs(config.bucket, bucket) - self.assertTrue(config.uniform_bucket_level_access_enabled) - self.assertEqual(config.uniform_bucket_level_access_locked_time, now) - self.assertTrue(config.bucket_policy_only_enabled) - self.assertEqual(config.bucket_policy_only_locked_time, now) - - def test_ctor_explicit_bpo(self): - import datetime - import pytz - - bucket = self._make_bucket() - now = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) - - config = pytest.deprecated_call( - self._make_one, - bucket, - bucket_policy_only_enabled=True, - bucket_policy_only_locked_time=now, - ) - - self.assertIs(config.bucket, bucket) - self.assertTrue(config.uniform_bucket_level_access_enabled) - self.assertEqual(config.uniform_bucket_level_access_locked_time, now) - self.assertTrue(config.bucket_policy_only_enabled) - self.assertEqual(config.bucket_policy_only_locked_time, now) - - def test_ctor_ubla_and_bpo_enabled(self): - bucket = self._make_bucket() - - with self.assertRaises(ValueError): - self._make_one( - bucket, - uniform_bucket_level_access_enabled=True, - bucket_policy_only_enabled=True, - ) - - def test_ctor_ubla_and_bpo_time(self): - import datetime - import pytz - - bucket = self._make_bucket() - now = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) - - with self.assertRaises(ValueError): - self._make_one( - bucket, - uniform_bucket_level_access_enabled=True, - uniform_bucket_level_access_locked_time=now, - bucket_policy_only_locked_time=now, - ) - - def test_from_api_repr_w_empty_resource(self): - klass = self._get_target_class() - bucket = self._make_bucket() - resource = {} - - config = klass.from_api_repr(resource, bucket) - - self.assertIs(config.bucket, bucket) - self.assertFalse(config.bucket_policy_only_enabled) - self.assertIsNone(config.bucket_policy_only_locked_time) - - def test_from_api_repr_w_empty_bpo(self): - klass = self._get_target_class() - bucket = self._make_bucket() - resource = {"uniformBucketLevelAccess": {}} - - config = klass.from_api_repr(resource, bucket) - - self.assertIs(config.bucket, bucket) - self.assertFalse(config.bucket_policy_only_enabled) - self.assertIsNone(config.bucket_policy_only_locked_time) - - def test_from_api_repr_w_disabled(self): - klass = self._get_target_class() - bucket = self._make_bucket() - resource = {"uniformBucketLevelAccess": {"enabled": False}} - - config = klass.from_api_repr(resource, bucket) - - self.assertIs(config.bucket, bucket) - self.assertFalse(config.bucket_policy_only_enabled) - self.assertIsNone(config.bucket_policy_only_locked_time) - - def test_from_api_repr_w_enabled(self): - import datetime - import pytz - from google.cloud._helpers import _datetime_to_rfc3339 - - klass = self._get_target_class() - bucket = self._make_bucket() - now = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) - resource = { - "uniformBucketLevelAccess": { - "enabled": True, - "lockedTime": _datetime_to_rfc3339(now), - } - } - - config = klass.from_api_repr(resource, bucket) - - self.assertIs(config.bucket, bucket) - self.assertTrue(config.uniform_bucket_level_access_enabled) - self.assertEqual(config.uniform_bucket_level_access_locked_time, now) - self.assertTrue(config.bucket_policy_only_enabled) - self.assertEqual(config.bucket_policy_only_locked_time, now) - - def test_uniform_bucket_level_access_enabled_setter(self): - bucket = self._make_bucket() - config = self._make_one(bucket) - - config.uniform_bucket_level_access_enabled = True - self.assertTrue(config.bucket_policy_only_enabled) - - self.assertTrue(config["uniformBucketLevelAccess"]["enabled"]) - bucket._patch_property.assert_called_once_with("iamConfiguration", config) - - def test_bucket_policy_only_enabled_setter(self): - bucket = self._make_bucket() - config = self._make_one(bucket) - - with pytest.deprecated_call(): - config.bucket_policy_only_enabled = True - - self.assertTrue(config.uniform_bucket_level_access_enabled) - self.assertTrue(config["uniformBucketLevelAccess"]["enabled"]) - bucket._patch_property.assert_called_once_with("iamConfiguration", config) - - -class Test_Bucket(unittest.TestCase): - @staticmethod - def _get_target_class(): - from google.cloud.storage.bucket import Bucket - - return Bucket - - @staticmethod - def _get_default_timeout(): - from google.cloud.storage.constants import _DEFAULT_TIMEOUT - - return _DEFAULT_TIMEOUT - - def _make_one(self, client=None, name=None, properties=None, user_project=None): - if client is None: - connection = _Connection() - client = _Client(connection) - if user_project is None: - bucket = self._get_target_class()(client, name=name) - else: - bucket = self._get_target_class()( - client, name=name, user_project=user_project - ) - bucket._properties = properties or {} - return bucket - - def test_ctor_w_invalid_name(self): - NAME = "#invalid" - with self.assertRaises(ValueError): - self._make_one(name=NAME) - - def test_ctor(self): - NAME = "name" - properties = {"key": "value"} - bucket = self._make_one(name=NAME, properties=properties) - self.assertEqual(bucket.name, NAME) - self.assertEqual(bucket._properties, properties) - self.assertEqual(list(bucket._changes), []) - self.assertFalse(bucket._acl.loaded) - self.assertIs(bucket._acl.bucket, bucket) - self.assertFalse(bucket._default_object_acl.loaded) - self.assertIs(bucket._default_object_acl.bucket, bucket) - self.assertEqual(list(bucket._label_removals), []) - self.assertIsNone(bucket.user_project) - - def test_ctor_w_user_project(self): - NAME = "name" - USER_PROJECT = "user-project-123" - connection = _Connection() - client = _Client(connection) - bucket = self._make_one(client, name=NAME, user_project=USER_PROJECT) - self.assertEqual(bucket.name, NAME) - self.assertEqual(bucket._properties, {}) - self.assertEqual(list(bucket._changes), []) - self.assertFalse(bucket._acl.loaded) - self.assertIs(bucket._acl.bucket, bucket) - self.assertFalse(bucket._default_object_acl.loaded) - self.assertIs(bucket._default_object_acl.bucket, bucket) - self.assertEqual(list(bucket._label_removals), []) - self.assertEqual(bucket.user_project, USER_PROJECT) - - def test_blob_wo_keys(self): - from google.cloud.storage.blob import Blob - - BUCKET_NAME = "BUCKET_NAME" - BLOB_NAME = "BLOB_NAME" - CHUNK_SIZE = 1024 * 1024 - - bucket = self._make_one(name=BUCKET_NAME) - blob = bucket.blob(BLOB_NAME, chunk_size=CHUNK_SIZE) - self.assertIsInstance(blob, Blob) - self.assertIs(blob.bucket, bucket) - self.assertIs(blob.client, bucket.client) - self.assertEqual(blob.name, BLOB_NAME) - self.assertEqual(blob.chunk_size, CHUNK_SIZE) - self.assertIsNone(blob._encryption_key) - self.assertIsNone(blob.kms_key_name) - - def test_blob_w_encryption_key(self): - from google.cloud.storage.blob import Blob - - BUCKET_NAME = "BUCKET_NAME" - BLOB_NAME = "BLOB_NAME" - CHUNK_SIZE = 1024 * 1024 - KEY = b"01234567890123456789012345678901" # 32 bytes - - bucket = self._make_one(name=BUCKET_NAME) - blob = bucket.blob(BLOB_NAME, chunk_size=CHUNK_SIZE, encryption_key=KEY) - self.assertIsInstance(blob, Blob) - self.assertIs(blob.bucket, bucket) - self.assertIs(blob.client, bucket.client) - self.assertEqual(blob.name, BLOB_NAME) - self.assertEqual(blob.chunk_size, CHUNK_SIZE) - self.assertEqual(blob._encryption_key, KEY) - self.assertIsNone(blob.kms_key_name) - - def test_blob_w_generation(self): - from google.cloud.storage.blob import Blob - - BUCKET_NAME = "BUCKET_NAME" - BLOB_NAME = "BLOB_NAME" - GENERATION = 123 - - bucket = self._make_one(name=BUCKET_NAME) - blob = bucket.blob(BLOB_NAME, generation=GENERATION) - self.assertIsInstance(blob, Blob) - self.assertIs(blob.bucket, bucket) - self.assertIs(blob.client, bucket.client) - self.assertEqual(blob.name, BLOB_NAME) - self.assertEqual(blob.generation, GENERATION) - - def test_blob_w_kms_key_name(self): - from google.cloud.storage.blob import Blob - - BUCKET_NAME = "BUCKET_NAME" - BLOB_NAME = "BLOB_NAME" - CHUNK_SIZE = 1024 * 1024 - KMS_RESOURCE = ( - "projects/test-project-123/" - "locations/us/" - "keyRings/test-ring/" - "cryptoKeys/test-key" - ) - - bucket = self._make_one(name=BUCKET_NAME) - blob = bucket.blob(BLOB_NAME, chunk_size=CHUNK_SIZE, kms_key_name=KMS_RESOURCE) - self.assertIsInstance(blob, Blob) - self.assertIs(blob.bucket, bucket) - self.assertIs(blob.client, bucket.client) - self.assertEqual(blob.name, BLOB_NAME) - self.assertEqual(blob.chunk_size, CHUNK_SIZE) - self.assertIsNone(blob._encryption_key) - self.assertEqual(blob.kms_key_name, KMS_RESOURCE) - - def test_notification_defaults(self): - from google.cloud.storage.notification import BucketNotification - from google.cloud.storage.notification import NONE_PAYLOAD_FORMAT - - PROJECT = "PROJECT" - BUCKET_NAME = "BUCKET_NAME" - TOPIC_NAME = "TOPIC_NAME" - client = _Client(_Connection(), project=PROJECT) - bucket = self._make_one(client, name=BUCKET_NAME) - - notification = bucket.notification(TOPIC_NAME) - - self.assertIsInstance(notification, BucketNotification) - self.assertIs(notification.bucket, bucket) - self.assertEqual(notification.topic_project, PROJECT) - self.assertIsNone(notification.custom_attributes) - self.assertIsNone(notification.event_types) - self.assertIsNone(notification.blob_name_prefix) - self.assertEqual(notification.payload_format, NONE_PAYLOAD_FORMAT) - - def test_notification_explicit(self): - from google.cloud.storage.notification import ( - BucketNotification, - OBJECT_FINALIZE_EVENT_TYPE, - OBJECT_DELETE_EVENT_TYPE, - JSON_API_V1_PAYLOAD_FORMAT, - ) - - PROJECT = "PROJECT" - BUCKET_NAME = "BUCKET_NAME" - TOPIC_NAME = "TOPIC_NAME" - TOPIC_ALT_PROJECT = "topic-project-456" - CUSTOM_ATTRIBUTES = {"attr1": "value1", "attr2": "value2"} - EVENT_TYPES = [OBJECT_FINALIZE_EVENT_TYPE, OBJECT_DELETE_EVENT_TYPE] - BLOB_NAME_PREFIX = "blob-name-prefix/" - client = _Client(_Connection(), project=PROJECT) - bucket = self._make_one(client, name=BUCKET_NAME) - - notification = bucket.notification( - TOPIC_NAME, - topic_project=TOPIC_ALT_PROJECT, - custom_attributes=CUSTOM_ATTRIBUTES, - event_types=EVENT_TYPES, - blob_name_prefix=BLOB_NAME_PREFIX, - payload_format=JSON_API_V1_PAYLOAD_FORMAT, - ) - - self.assertIsInstance(notification, BucketNotification) - self.assertIs(notification.bucket, bucket) - self.assertEqual(notification.topic_project, TOPIC_ALT_PROJECT) - self.assertEqual(notification.custom_attributes, CUSTOM_ATTRIBUTES) - self.assertEqual(notification.event_types, EVENT_TYPES) - self.assertEqual(notification.blob_name_prefix, BLOB_NAME_PREFIX) - self.assertEqual(notification.payload_format, JSON_API_V1_PAYLOAD_FORMAT) - - def test_bucket_name_value(self): - BUCKET_NAME = "bucket-name" - self._make_one(name=BUCKET_NAME) - - bad_start_bucket_name = "/testing123" - with self.assertRaises(ValueError): - self._make_one(name=bad_start_bucket_name) - - bad_end_bucket_name = "testing123/" - with self.assertRaises(ValueError): - self._make_one(name=bad_end_bucket_name) - - def test_user_project(self): - BUCKET_NAME = "name" - USER_PROJECT = "user-project-123" - bucket = self._make_one(name=BUCKET_NAME) - bucket._user_project = USER_PROJECT - self.assertEqual(bucket.user_project, USER_PROJECT) - - def test_exists_miss(self): - from google.cloud.exceptions import NotFound - - class _FakeConnection(object): - - _called_with = [] - - @classmethod - def api_request(cls, *args, **kwargs): - cls._called_with.append((args, kwargs)) - raise NotFound(args) - - BUCKET_NAME = "bucket-name" - bucket = self._make_one(name=BUCKET_NAME) - client = _Client(_FakeConnection) - self.assertFalse(bucket.exists(client=client, timeout=42)) - expected_called_kwargs = { - "method": "GET", - "path": bucket.path, - "query_params": {"fields": "name"}, - "_target_object": None, - "timeout": 42, - } - expected_cw = [((), expected_called_kwargs)] - self.assertEqual(_FakeConnection._called_with, expected_cw) - - def test_exists_hit_w_user_project(self): - USER_PROJECT = "user-project-123" - - class _FakeConnection(object): - - _called_with = [] - - @classmethod - def api_request(cls, *args, **kwargs): - cls._called_with.append((args, kwargs)) - # exists() does not use the return value - return object() - - BUCKET_NAME = "bucket-name" - bucket = self._make_one(name=BUCKET_NAME, user_project=USER_PROJECT) - client = _Client(_FakeConnection) - self.assertTrue(bucket.exists(client=client)) - expected_called_kwargs = { - "method": "GET", - "path": bucket.path, - "query_params": {"fields": "name", "userProject": USER_PROJECT}, - "_target_object": None, - "timeout": self._get_default_timeout(), - } - expected_cw = [((), expected_called_kwargs)] - self.assertEqual(_FakeConnection._called_with, expected_cw) - - def test_acl_property(self): - from google.cloud.storage.acl import BucketACL - - bucket = self._make_one() - acl = bucket.acl - self.assertIsInstance(acl, BucketACL) - self.assertIs(acl, bucket._acl) - - def test_default_object_acl_property(self): - from google.cloud.storage.acl import DefaultObjectACL - - bucket = self._make_one() - acl = bucket.default_object_acl - self.assertIsInstance(acl, DefaultObjectACL) - self.assertIs(acl, bucket._default_object_acl) - - def test_path_no_name(self): - bucket = self._make_one() - self.assertRaises(ValueError, getattr, bucket, "path") - - def test_path_w_name(self): - NAME = "name" - bucket = self._make_one(name=NAME) - self.assertEqual(bucket.path, "/b/%s" % NAME) - - def test_get_blob_miss(self): - NAME = "name" - NONESUCH = "nonesuch" - connection = _Connection() - client = _Client(connection) - bucket = self._make_one(name=NAME) - result = bucket.get_blob(NONESUCH, client=client, timeout=42) - self.assertIsNone(result) - (kw,) = connection._requested - self.assertEqual(kw["method"], "GET") - self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, NONESUCH)) - self.assertEqual(kw["timeout"], 42) - - def test_get_blob_hit_w_user_project(self): - NAME = "name" - BLOB_NAME = "blob-name" - USER_PROJECT = "user-project-123" - connection = _Connection({"name": BLOB_NAME}) - client = _Client(connection) - bucket = self._make_one(name=NAME, user_project=USER_PROJECT) - blob = bucket.get_blob(BLOB_NAME, client=client) - self.assertIs(blob.bucket, bucket) - self.assertEqual(blob.name, BLOB_NAME) - (kw,) = connection._requested - expected_qp = {"userProject": USER_PROJECT, "projection": "noAcl"} - self.assertEqual(kw["method"], "GET") - self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) - self.assertEqual(kw["query_params"], expected_qp) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - - def test_get_blob_hit_w_generation(self): - NAME = "name" - BLOB_NAME = "blob-name" - GENERATION = 1512565576797178 - connection = _Connection({"name": BLOB_NAME, "generation": GENERATION}) - client = _Client(connection) - bucket = self._make_one(name=NAME) - blob = bucket.get_blob(BLOB_NAME, client=client, generation=GENERATION) - self.assertIs(blob.bucket, bucket) - self.assertEqual(blob.name, BLOB_NAME) - self.assertEqual(blob.generation, GENERATION) - (kw,) = connection._requested - expected_qp = {"generation": GENERATION, "projection": "noAcl"} - self.assertEqual(kw["method"], "GET") - self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) - self.assertEqual(kw["query_params"], expected_qp) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - - def test_get_blob_hit_with_kwargs(self): - from google.cloud.storage.blob import _get_encryption_headers - - NAME = "name" - BLOB_NAME = "blob-name" - CHUNK_SIZE = 1024 * 1024 - KEY = b"01234567890123456789012345678901" # 32 bytes - - connection = _Connection({"name": BLOB_NAME}) - client = _Client(connection) - bucket = self._make_one(name=NAME) - blob = bucket.get_blob( - BLOB_NAME, client=client, encryption_key=KEY, chunk_size=CHUNK_SIZE - ) - self.assertIs(blob.bucket, bucket) - self.assertEqual(blob.name, BLOB_NAME) - (kw,) = connection._requested - self.assertEqual(kw["method"], "GET") - self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) - self.assertEqual(kw["headers"], _get_encryption_headers(KEY)) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - self.assertEqual(blob.chunk_size, CHUNK_SIZE) - self.assertEqual(blob._encryption_key, KEY) - - def test_list_blobs_defaults(self): - NAME = "name" - connection = _Connection({"items": []}) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - iterator = bucket.list_blobs() - blobs = list(iterator) - self.assertEqual(blobs, []) - (kw,) = connection._requested - self.assertEqual(kw["method"], "GET") - self.assertEqual(kw["path"], "/b/%s/o" % NAME) - self.assertEqual(kw["query_params"], {"projection": "noAcl"}) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - - def test_list_blobs_w_all_arguments_and_user_project(self): - NAME = "name" - USER_PROJECT = "user-project-123" - MAX_RESULTS = 10 - PAGE_TOKEN = "ABCD" - PREFIX = "subfolder" - DELIMITER = "/" - VERSIONS = True - PROJECTION = "full" - FIELDS = "items/contentLanguage,nextPageToken" - EXPECTED = { - "maxResults": 10, - "pageToken": PAGE_TOKEN, - "prefix": PREFIX, - "delimiter": DELIMITER, - "versions": VERSIONS, - "projection": PROJECTION, - "fields": FIELDS, - "userProject": USER_PROJECT, - } - connection = _Connection({"items": []}) - client = _Client(connection) - bucket = self._make_one(name=NAME, user_project=USER_PROJECT) - iterator = bucket.list_blobs( - max_results=MAX_RESULTS, - page_token=PAGE_TOKEN, - prefix=PREFIX, - delimiter=DELIMITER, - versions=VERSIONS, - projection=PROJECTION, - fields=FIELDS, - client=client, - timeout=42, - ) - blobs = list(iterator) - self.assertEqual(blobs, []) - (kw,) = connection._requested - self.assertEqual(kw["method"], "GET") - self.assertEqual(kw["path"], "/b/%s/o" % NAME) - self.assertEqual(kw["query_params"], EXPECTED) - self.assertEqual(kw["timeout"], 42) - - def test_list_notifications(self): - from google.cloud.storage.notification import BucketNotification - from google.cloud.storage.notification import _TOPIC_REF_FMT - from google.cloud.storage.notification import ( - JSON_API_V1_PAYLOAD_FORMAT, - NONE_PAYLOAD_FORMAT, - ) - - NAME = "name" - - topic_refs = [("my-project-123", "topic-1"), ("other-project-456", "topic-2")] - - resources = [ - { - "topic": _TOPIC_REF_FMT.format(*topic_refs[0]), - "id": "1", - "etag": "DEADBEEF", - "selfLink": "https://example.com/notification/1", - "payload_format": NONE_PAYLOAD_FORMAT, - }, - { - "topic": _TOPIC_REF_FMT.format(*topic_refs[1]), - "id": "2", - "etag": "FACECABB", - "selfLink": "https://example.com/notification/2", - "payload_format": JSON_API_V1_PAYLOAD_FORMAT, - }, - ] - connection = _Connection({"items": resources}) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - - notifications = list(bucket.list_notifications(timeout=42)) - - req_args = client._connection._requested[0] - self.assertEqual(req_args.get("timeout"), 42) - - self.assertEqual(len(notifications), len(resources)) - for notification, resource, topic_ref in zip( - notifications, resources, topic_refs - ): - self.assertIsInstance(notification, BucketNotification) - self.assertEqual(notification.topic_project, topic_ref[0]) - self.assertEqual(notification.topic_name, topic_ref[1]) - self.assertEqual(notification.notification_id, resource["id"]) - self.assertEqual(notification.etag, resource["etag"]) - self.assertEqual(notification.self_link, resource["selfLink"]) - self.assertEqual( - notification.custom_attributes, resource.get("custom_attributes") - ) - self.assertEqual(notification.event_types, resource.get("event_types")) - self.assertEqual( - notification.blob_name_prefix, resource.get("blob_name_prefix") - ) - self.assertEqual( - notification.payload_format, resource.get("payload_format") - ) - - def test_get_notification(self): - from google.cloud.storage.notification import _TOPIC_REF_FMT - from google.cloud.storage.notification import JSON_API_V1_PAYLOAD_FORMAT - - NAME = "name" - ETAG = "FACECABB" - NOTIFICATION_ID = "1" - SELF_LINK = "https://example.com/notification/1" - resources = { - "topic": _TOPIC_REF_FMT.format("my-project-123", "topic-1"), - "id": NOTIFICATION_ID, - "etag": ETAG, - "selfLink": SELF_LINK, - "payload_format": JSON_API_V1_PAYLOAD_FORMAT, - } - - connection = _make_connection(resources) - client = _Client(connection, project="my-project-123") - bucket = self._make_one(client=client, name=NAME) - notification = bucket.get_notification(notification_id=NOTIFICATION_ID) - - self.assertEqual(notification.notification_id, NOTIFICATION_ID) - self.assertEqual(notification.etag, ETAG) - self.assertEqual(notification.self_link, SELF_LINK) - self.assertIsNone(notification.custom_attributes) - self.assertIsNone(notification.event_types) - self.assertIsNone(notification.blob_name_prefix) - self.assertEqual(notification.payload_format, JSON_API_V1_PAYLOAD_FORMAT) - - def test_get_notification_miss(self): - from google.cloud.exceptions import NotFound - - response = NotFound("testing") - connection = _make_connection(response) - client = _Client(connection, project="my-project-123") - bucket = self._make_one(client=client, name="name") - with self.assertRaises(NotFound): - bucket.get_notification(notification_id="1") - - def test_delete_miss(self): - from google.cloud.exceptions import NotFound - - NAME = "name" - connection = _Connection() - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - self.assertRaises(NotFound, bucket.delete) - expected_cw = [ - { - "method": "DELETE", - "path": bucket.path, - "query_params": {}, - "_target_object": None, - "timeout": self._get_default_timeout(), - } - ] - self.assertEqual(connection._deleted_buckets, expected_cw) - - def test_delete_hit_with_user_project(self): - NAME = "name" - USER_PROJECT = "user-project-123" - GET_BLOBS_RESP = {"items": []} - connection = _Connection(GET_BLOBS_RESP) - connection._delete_bucket = True - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME, user_project=USER_PROJECT) - result = bucket.delete(force=True, timeout=42) - self.assertIsNone(result) - expected_cw = [ - { - "method": "DELETE", - "path": bucket.path, - "_target_object": None, - "query_params": {"userProject": USER_PROJECT}, - "timeout": 42, - } - ] - self.assertEqual(connection._deleted_buckets, expected_cw) - - def test_delete_force_delete_blobs(self): - NAME = "name" - BLOB_NAME1 = "blob-name1" - BLOB_NAME2 = "blob-name2" - GET_BLOBS_RESP = {"items": [{"name": BLOB_NAME1}, {"name": BLOB_NAME2}]} - DELETE_BLOB1_RESP = DELETE_BLOB2_RESP = {} - connection = _Connection(GET_BLOBS_RESP, DELETE_BLOB1_RESP, DELETE_BLOB2_RESP) - connection._delete_bucket = True - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - result = bucket.delete(force=True) - self.assertIsNone(result) - expected_cw = [ - { - "method": "DELETE", - "path": bucket.path, - "query_params": {}, - "_target_object": None, - "timeout": self._get_default_timeout(), - } - ] - self.assertEqual(connection._deleted_buckets, expected_cw) - - def test_delete_force_miss_blobs(self): - NAME = "name" - BLOB_NAME = "blob-name1" - GET_BLOBS_RESP = {"items": [{"name": BLOB_NAME}]} - # Note the connection does not have a response for the blob. - connection = _Connection(GET_BLOBS_RESP) - connection._delete_bucket = True - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - result = bucket.delete(force=True) - self.assertIsNone(result) - expected_cw = [ - { - "method": "DELETE", - "path": bucket.path, - "query_params": {}, - "_target_object": None, - "timeout": self._get_default_timeout(), - } - ] - self.assertEqual(connection._deleted_buckets, expected_cw) - - def test_delete_too_many(self): - NAME = "name" - BLOB_NAME1 = "blob-name1" - BLOB_NAME2 = "blob-name2" - GET_BLOBS_RESP = {"items": [{"name": BLOB_NAME1}, {"name": BLOB_NAME2}]} - connection = _Connection(GET_BLOBS_RESP) - connection._delete_bucket = True - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - - # Make the Bucket refuse to delete with 2 objects. - bucket._MAX_OBJECTS_FOR_ITERATION = 1 - self.assertRaises(ValueError, bucket.delete, force=True) - self.assertEqual(connection._deleted_buckets, []) - - def test_delete_blob_miss(self): - from google.cloud.exceptions import NotFound - - NAME = "name" - NONESUCH = "nonesuch" - connection = _Connection() - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - self.assertRaises(NotFound, bucket.delete_blob, NONESUCH) - (kw,) = connection._requested - self.assertEqual(kw["method"], "DELETE") - self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, NONESUCH)) - self.assertEqual(kw["query_params"], {}) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - - def test_delete_blob_hit_with_user_project(self): - NAME = "name" - BLOB_NAME = "blob-name" - USER_PROJECT = "user-project-123" - connection = _Connection({}) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME, user_project=USER_PROJECT) - result = bucket.delete_blob(BLOB_NAME, timeout=42) - self.assertIsNone(result) - (kw,) = connection._requested - self.assertEqual(kw["method"], "DELETE") - self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) - self.assertEqual(kw["query_params"], {"userProject": USER_PROJECT}) - self.assertEqual(kw["timeout"], 42) - - def test_delete_blob_hit_with_generation(self): - NAME = "name" - BLOB_NAME = "blob-name" - GENERATION = 1512565576797178 - connection = _Connection({}) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - result = bucket.delete_blob(BLOB_NAME, generation=GENERATION) - self.assertIsNone(result) - (kw,) = connection._requested - self.assertEqual(kw["method"], "DELETE") - self.assertEqual(kw["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) - self.assertEqual(kw["query_params"], {"generation": GENERATION}) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - - def test_delete_blobs_empty(self): - NAME = "name" - connection = _Connection() - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - bucket.delete_blobs([]) - self.assertEqual(connection._requested, []) - - def test_delete_blobs_hit_w_user_project(self): - NAME = "name" - BLOB_NAME = "blob-name" - USER_PROJECT = "user-project-123" - connection = _Connection({}) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME, user_project=USER_PROJECT) - bucket.delete_blobs([BLOB_NAME], timeout=42) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "DELETE") - self.assertEqual(kw[0]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) - self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) - self.assertEqual(kw[0]["timeout"], 42) - - def test_delete_blobs_miss_no_on_error(self): - from google.cloud.exceptions import NotFound - - NAME = "name" - BLOB_NAME = "blob-name" - NONESUCH = "nonesuch" - connection = _Connection({}) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - self.assertRaises(NotFound, bucket.delete_blobs, [BLOB_NAME, NONESUCH]) - kw = connection._requested - self.assertEqual(len(kw), 2) - self.assertEqual(kw[0]["method"], "DELETE") - self.assertEqual(kw[0]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - self.assertEqual(kw[1]["method"], "DELETE") - self.assertEqual(kw[1]["path"], "/b/%s/o/%s" % (NAME, NONESUCH)) - self.assertEqual(kw[1]["timeout"], self._get_default_timeout()) - - def test_delete_blobs_miss_w_on_error(self): - NAME = "name" - BLOB_NAME = "blob-name" - NONESUCH = "nonesuch" - connection = _Connection({}) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - errors = [] - bucket.delete_blobs([BLOB_NAME, NONESUCH], errors.append) - self.assertEqual(errors, [NONESUCH]) - kw = connection._requested - self.assertEqual(len(kw), 2) - self.assertEqual(kw[0]["method"], "DELETE") - self.assertEqual(kw[0]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - self.assertEqual(kw[1]["method"], "DELETE") - self.assertEqual(kw[1]["path"], "/b/%s/o/%s" % (NAME, NONESUCH)) - self.assertEqual(kw[1]["timeout"], self._get_default_timeout()) - - @staticmethod - def _make_blob(bucket_name, blob_name): - from google.cloud.storage.blob import Blob - - blob = mock.create_autospec(Blob) - blob.name = blob_name - blob.path = "/b/{}/o/{}".format(bucket_name, blob_name) - return blob - - def test_copy_blobs_wo_name(self): - SOURCE = "source" - DEST = "dest" - BLOB_NAME = "blob-name" - connection = _Connection({}) - client = _Client(connection) - source = self._make_one(client=client, name=SOURCE) - dest = self._make_one(client=client, name=DEST) - blob = self._make_blob(SOURCE, BLOB_NAME) - - new_blob = source.copy_blob(blob, dest, timeout=42) - - self.assertIs(new_blob.bucket, dest) - self.assertEqual(new_blob.name, BLOB_NAME) - - (kw,) = connection._requested - COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( - SOURCE, BLOB_NAME, DEST, BLOB_NAME - ) - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], COPY_PATH) - self.assertEqual(kw["query_params"], {}) - self.assertEqual(kw["timeout"], 42) - - def test_copy_blobs_source_generation(self): - SOURCE = "source" - DEST = "dest" - BLOB_NAME = "blob-name" - GENERATION = 1512565576797178 - - connection = _Connection({}) - client = _Client(connection) - source = self._make_one(client=client, name=SOURCE) - dest = self._make_one(client=client, name=DEST) - blob = self._make_blob(SOURCE, BLOB_NAME) - - new_blob = source.copy_blob(blob, dest, source_generation=GENERATION) - - self.assertIs(new_blob.bucket, dest) - self.assertEqual(new_blob.name, BLOB_NAME) - - (kw,) = connection._requested - COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( - SOURCE, BLOB_NAME, DEST, BLOB_NAME - ) - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], COPY_PATH) - self.assertEqual(kw["query_params"], {"sourceGeneration": GENERATION}) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - - def test_copy_blobs_preserve_acl(self): - from google.cloud.storage.acl import ObjectACL - - SOURCE = "source" - DEST = "dest" - BLOB_NAME = "blob-name" - NEW_NAME = "new_name" - - connection = _Connection({}, {}) - client = _Client(connection) - source = self._make_one(client=client, name=SOURCE) - dest = self._make_one(client=client, name=DEST) - blob = self._make_blob(SOURCE, BLOB_NAME) - - new_blob = source.copy_blob( - blob, dest, NEW_NAME, client=client, preserve_acl=False - ) - - self.assertIs(new_blob.bucket, dest) - self.assertEqual(new_blob.name, NEW_NAME) - self.assertIsInstance(new_blob.acl, ObjectACL) - - kw1, kw2 = connection._requested - COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( - SOURCE, BLOB_NAME, DEST, NEW_NAME - ) - NEW_BLOB_PATH = "/b/{}/o/{}".format(DEST, NEW_NAME) - - self.assertEqual(kw1["method"], "POST") - self.assertEqual(kw1["path"], COPY_PATH) - self.assertEqual(kw1["query_params"], {}) - self.assertEqual(kw1["timeout"], self._get_default_timeout()) - - self.assertEqual(kw2["method"], "PATCH") - self.assertEqual(kw2["path"], NEW_BLOB_PATH) - self.assertEqual(kw2["query_params"], {"projection": "full"}) - self.assertEqual(kw2["timeout"], self._get_default_timeout()) - - def test_copy_blobs_w_name_and_user_project(self): - SOURCE = "source" - DEST = "dest" - BLOB_NAME = "blob-name" - NEW_NAME = "new_name" - USER_PROJECT = "user-project-123" - connection = _Connection({}) - client = _Client(connection) - source = self._make_one(client=client, name=SOURCE, user_project=USER_PROJECT) - dest = self._make_one(client=client, name=DEST) - blob = self._make_blob(SOURCE, BLOB_NAME) - - new_blob = source.copy_blob(blob, dest, NEW_NAME) - - self.assertIs(new_blob.bucket, dest) - self.assertEqual(new_blob.name, NEW_NAME) - - COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( - SOURCE, BLOB_NAME, DEST, NEW_NAME - ) - (kw,) = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], COPY_PATH) - self.assertEqual(kw["query_params"], {"userProject": USER_PROJECT}) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - - def test_rename_blob(self): - BUCKET_NAME = "BUCKET_NAME" - BLOB_NAME = "blob-name" - NEW_BLOB_NAME = "new-blob-name" - DATA = {"name": NEW_BLOB_NAME} - connection = _Connection(DATA) - client = _Client(connection) - bucket = self._make_one(client=client, name=BUCKET_NAME) - blob = self._make_blob(BUCKET_NAME, BLOB_NAME) - - renamed_blob = bucket.rename_blob( - blob, NEW_BLOB_NAME, client=client, timeout=42 - ) - - self.assertIs(renamed_blob.bucket, bucket) - self.assertEqual(renamed_blob.name, NEW_BLOB_NAME) - - COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( - BUCKET_NAME, BLOB_NAME, BUCKET_NAME, NEW_BLOB_NAME - ) - (kw,) = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], COPY_PATH) - self.assertEqual(kw["query_params"], {}) - self.assertEqual(kw["timeout"], 42) - - blob.delete.assert_called_once_with(client=client, timeout=42) - - def test_rename_blob_to_itself(self): - BUCKET_NAME = "BUCKET_NAME" - BLOB_NAME = "blob-name" - DATA = {"name": BLOB_NAME} - connection = _Connection(DATA) - client = _Client(connection) - bucket = self._make_one(client=client, name=BUCKET_NAME) - blob = self._make_blob(BUCKET_NAME, BLOB_NAME) - - renamed_blob = bucket.rename_blob(blob, BLOB_NAME) - - self.assertIs(renamed_blob.bucket, bucket) - self.assertEqual(renamed_blob.name, BLOB_NAME) - - COPY_PATH = "/b/{}/o/{}/copyTo/b/{}/o/{}".format( - BUCKET_NAME, BLOB_NAME, BUCKET_NAME, BLOB_NAME - ) - (kw,) = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], COPY_PATH) - self.assertEqual(kw["query_params"], {}) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - - blob.delete.assert_not_called() - - def test_etag(self): - ETAG = "ETAG" - properties = {"etag": ETAG} - bucket = self._make_one(properties=properties) - self.assertEqual(bucket.etag, ETAG) - - def test_id(self): - ID = "ID" - properties = {"id": ID} - bucket = self._make_one(properties=properties) - self.assertEqual(bucket.id, ID) - - def test_location_getter(self): - NAME = "name" - before = {"location": "AS"} - bucket = self._make_one(name=NAME, properties=before) - self.assertEqual(bucket.location, "AS") - - @mock.patch("warnings.warn") - def test_location_setter(self, mock_warn): - from google.cloud.storage import bucket as bucket_module - - NAME = "name" - bucket = self._make_one(name=NAME) - self.assertIsNone(bucket.location) - bucket.location = "AS" - self.assertEqual(bucket.location, "AS") - self.assertTrue("location" in bucket._changes) - mock_warn.assert_called_once_with( - bucket_module._LOCATION_SETTER_MESSAGE, DeprecationWarning, stacklevel=2 - ) - - def test_iam_configuration_policy_missing(self): - from google.cloud.storage.bucket import IAMConfiguration - - NAME = "name" - bucket = self._make_one(name=NAME) - - config = bucket.iam_configuration - - self.assertIsInstance(config, IAMConfiguration) - self.assertIs(config.bucket, bucket) - self.assertFalse(config.bucket_policy_only_enabled) - self.assertIsNone(config.bucket_policy_only_locked_time) - - def test_iam_configuration_policy_w_entry(self): - import datetime - import pytz - from google.cloud._helpers import _datetime_to_rfc3339 - from google.cloud.storage.bucket import IAMConfiguration - - now = datetime.datetime.utcnow().replace(tzinfo=pytz.UTC) - NAME = "name" - properties = { - "iamConfiguration": { - "uniformBucketLevelAccess": { - "enabled": True, - "lockedTime": _datetime_to_rfc3339(now), - } - } - } - bucket = self._make_one(name=NAME, properties=properties) - - config = bucket.iam_configuration - - self.assertIsInstance(config, IAMConfiguration) - self.assertIs(config.bucket, bucket) - self.assertTrue(config.uniform_bucket_level_access_enabled) - self.assertEqual(config.uniform_bucket_level_access_locked_time, now) - - def test_lifecycle_rules_getter_unknown_action_type(self): - NAME = "name" - BOGUS_RULE = {"action": {"type": "Bogus"}, "condition": {"age": 42}} - rules = [BOGUS_RULE] - properties = {"lifecycle": {"rule": rules}} - bucket = self._make_one(name=NAME, properties=properties) - - with self.assertRaises(ValueError): - list(bucket.lifecycle_rules) - - def test_lifecycle_rules_getter(self): - from google.cloud.storage.bucket import ( - LifecycleRuleDelete, - LifecycleRuleSetStorageClass, - ) - - NAME = "name" - DELETE_RULE = {"action": {"type": "Delete"}, "condition": {"age": 42}} - SSC_RULE = { - "action": {"type": "SetStorageClass", "storageClass": "NEARLINE"}, - "condition": {"isLive": False}, - } - rules = [DELETE_RULE, SSC_RULE] - properties = {"lifecycle": {"rule": rules}} - bucket = self._make_one(name=NAME, properties=properties) - - found = list(bucket.lifecycle_rules) - - delete_rule = found[0] - self.assertIsInstance(delete_rule, LifecycleRuleDelete) - self.assertEqual(dict(delete_rule), DELETE_RULE) - - ssc_rule = found[1] - self.assertIsInstance(ssc_rule, LifecycleRuleSetStorageClass) - self.assertEqual(dict(ssc_rule), SSC_RULE) - - def test_lifecycle_rules_setter_w_dicts(self): - NAME = "name" - DELETE_RULE = {"action": {"type": "Delete"}, "condition": {"age": 42}} - SSC_RULE = { - "action": {"type": "SetStorageClass", "storageClass": "NEARLINE"}, - "condition": {"isLive": False}, - } - rules = [DELETE_RULE, SSC_RULE] - bucket = self._make_one(name=NAME) - self.assertEqual(list(bucket.lifecycle_rules), []) - - bucket.lifecycle_rules = rules - - self.assertEqual([dict(rule) for rule in bucket.lifecycle_rules], rules) - self.assertTrue("lifecycle" in bucket._changes) - - def test_lifecycle_rules_setter_w_helpers(self): - from google.cloud.storage.bucket import ( - LifecycleRuleDelete, - LifecycleRuleSetStorageClass, - ) - - NAME = "name" - DELETE_RULE = {"action": {"type": "Delete"}, "condition": {"age": 42}} - SSC_RULE = { - "action": {"type": "SetStorageClass", "storageClass": "NEARLINE"}, - "condition": {"isLive": False}, - } - rules = [DELETE_RULE, SSC_RULE] - bucket = self._make_one(name=NAME) - self.assertEqual(list(bucket.lifecycle_rules), []) - - bucket.lifecycle_rules = [ - LifecycleRuleDelete(age=42), - LifecycleRuleSetStorageClass("NEARLINE", is_live=False), - ] - - self.assertEqual([dict(rule) for rule in bucket.lifecycle_rules], rules) - self.assertTrue("lifecycle" in bucket._changes) - - def test_clear_lifecycle_rules(self): - NAME = "name" - DELETE_RULE = {"action": {"type": "Delete"}, "condition": {"age": 42}} - SSC_RULE = { - "action": {"type": "SetStorageClass", "storageClass": "NEARLINE"}, - "condition": {"isLive": False}, - } - rules = [DELETE_RULE, SSC_RULE] - bucket = self._make_one(name=NAME) - bucket._properties["lifecycle"] = {"rule": rules} - self.assertEqual(list(bucket.lifecycle_rules), rules) - - bucket.clear_lifecyle_rules() - - self.assertEqual(list(bucket.lifecycle_rules), []) - self.assertTrue("lifecycle" in bucket._changes) - - def test_add_lifecycle_delete_rule(self): - NAME = "name" - DELETE_RULE = {"action": {"type": "Delete"}, "condition": {"age": 42}} - rules = [DELETE_RULE] - bucket = self._make_one(name=NAME) - self.assertEqual(list(bucket.lifecycle_rules), []) - - bucket.add_lifecycle_delete_rule(age=42) - - self.assertEqual([dict(rule) for rule in bucket.lifecycle_rules], rules) - self.assertTrue("lifecycle" in bucket._changes) - - def test_add_lifecycle_set_storage_class_rule(self): - NAME = "name" - SSC_RULE = { - "action": {"type": "SetStorageClass", "storageClass": "NEARLINE"}, - "condition": {"isLive": False}, - } - rules = [SSC_RULE] - bucket = self._make_one(name=NAME) - self.assertEqual(list(bucket.lifecycle_rules), []) - - bucket.add_lifecycle_set_storage_class_rule("NEARLINE", is_live=False) - - self.assertEqual([dict(rule) for rule in bucket.lifecycle_rules], rules) - self.assertTrue("lifecycle" in bucket._changes) - - def test_cors_getter(self): - NAME = "name" - CORS_ENTRY = { - "maxAgeSeconds": 1234, - "method": ["OPTIONS", "GET"], - "origin": ["127.0.0.1"], - "responseHeader": ["Content-Type"], - } - properties = {"cors": [CORS_ENTRY, {}]} - bucket = self._make_one(name=NAME, properties=properties) - entries = bucket.cors - self.assertEqual(len(entries), 2) - self.assertEqual(entries[0], CORS_ENTRY) - self.assertEqual(entries[1], {}) - # Make sure it was a copy, not the same object. - self.assertIsNot(entries[0], CORS_ENTRY) - - def test_cors_setter(self): - NAME = "name" - CORS_ENTRY = { - "maxAgeSeconds": 1234, - "method": ["OPTIONS", "GET"], - "origin": ["127.0.0.1"], - "responseHeader": ["Content-Type"], - } - bucket = self._make_one(name=NAME) - - self.assertEqual(bucket.cors, []) - bucket.cors = [CORS_ENTRY] - self.assertEqual(bucket.cors, [CORS_ENTRY]) - self.assertTrue("cors" in bucket._changes) - - def test_default_kms_key_name_getter(self): - NAME = "name" - KMS_RESOURCE = ( - "projects/test-project-123/" - "locations/us/" - "keyRings/test-ring/" - "cryptoKeys/test-key" - ) - ENCRYPTION_CONFIG = {"defaultKmsKeyName": KMS_RESOURCE} - bucket = self._make_one(name=NAME) - self.assertIsNone(bucket.default_kms_key_name) - bucket._properties["encryption"] = ENCRYPTION_CONFIG - self.assertEqual(bucket.default_kms_key_name, KMS_RESOURCE) - - def test_default_kms_key_name_setter(self): - NAME = "name" - KMS_RESOURCE = ( - "projects/test-project-123/" - "locations/us/" - "keyRings/test-ring/" - "cryptoKeys/test-key" - ) - ENCRYPTION_CONFIG = {"defaultKmsKeyName": KMS_RESOURCE} - bucket = self._make_one(name=NAME) - bucket.default_kms_key_name = KMS_RESOURCE - self.assertEqual(bucket._properties["encryption"], ENCRYPTION_CONFIG) - self.assertTrue("encryption" in bucket._changes) - - def test_labels_getter(self): - NAME = "name" - LABELS = {"color": "red", "flavor": "cherry"} - properties = {"labels": LABELS} - bucket = self._make_one(name=NAME, properties=properties) - labels = bucket.labels - self.assertEqual(labels, LABELS) - # Make sure it was a copy, not the same object. - self.assertIsNot(labels, LABELS) - - def test_labels_setter(self): - NAME = "name" - LABELS = {"color": "red", "flavor": "cherry"} - bucket = self._make_one(name=NAME) - - self.assertEqual(bucket.labels, {}) - bucket.labels = LABELS - self.assertEqual(bucket.labels, LABELS) - self.assertIsNot(bucket._properties["labels"], LABELS) - self.assertIn("labels", bucket._changes) - - def test_labels_setter_with_nan(self): - NAME = "name" - LABELS = {"color": "red", "foo": float("nan")} - bucket = self._make_one(name=NAME) - - self.assertEqual(bucket.labels, {}) - bucket.labels = LABELS - value = bucket.labels["foo"] - self.assertIsInstance(value, str) - - def test_labels_setter_with_removal(self): - # Make sure the bucket labels look correct and follow the expected - # public structure. - bucket = self._make_one(name="name") - self.assertEqual(bucket.labels, {}) - bucket.labels = {"color": "red", "flavor": "cherry"} - self.assertEqual(bucket.labels, {"color": "red", "flavor": "cherry"}) - bucket.labels = {"color": "red"} - self.assertEqual(bucket.labels, {"color": "red"}) - - # Make sure that a patch call correctly removes the flavor label. - client = mock.NonCallableMock(spec=("_connection",)) - client._connection = mock.NonCallableMock(spec=("api_request",)) - bucket.patch(client=client) - client._connection.api_request.assert_called() - _, _, kwargs = client._connection.api_request.mock_calls[0] - self.assertEqual(len(kwargs["data"]["labels"]), 2) - self.assertEqual(kwargs["data"]["labels"]["color"], "red") - self.assertIsNone(kwargs["data"]["labels"]["flavor"]) - self.assertEqual(kwargs["timeout"], self._get_default_timeout()) - - # A second patch call should be a no-op for labels. - client._connection.api_request.reset_mock() - bucket.patch(client=client, timeout=42) - client._connection.api_request.assert_called() - _, _, kwargs = client._connection.api_request.mock_calls[0] - self.assertNotIn("labels", kwargs["data"]) - self.assertEqual(kwargs["timeout"], 42) - - def test_location_type_getter_unset(self): - bucket = self._make_one() - self.assertIsNone(bucket.location_type) - - def test_location_type_getter_set(self): - from google.cloud.storage.constants import REGION_LOCATION_TYPE - - properties = {"locationType": REGION_LOCATION_TYPE} - bucket = self._make_one(properties=properties) - self.assertEqual(bucket.location_type, REGION_LOCATION_TYPE) - - def test_get_logging_w_prefix(self): - NAME = "name" - LOG_BUCKET = "logs" - LOG_PREFIX = "pfx" - before = {"logging": {"logBucket": LOG_BUCKET, "logObjectPrefix": LOG_PREFIX}} - bucket = self._make_one(name=NAME, properties=before) - info = bucket.get_logging() - self.assertEqual(info["logBucket"], LOG_BUCKET) - self.assertEqual(info["logObjectPrefix"], LOG_PREFIX) - - def test_enable_logging_defaults(self): - NAME = "name" - LOG_BUCKET = "logs" - before = {"logging": None} - bucket = self._make_one(name=NAME, properties=before) - self.assertIsNone(bucket.get_logging()) - bucket.enable_logging(LOG_BUCKET) - info = bucket.get_logging() - self.assertEqual(info["logBucket"], LOG_BUCKET) - self.assertEqual(info["logObjectPrefix"], "") - - def test_enable_logging(self): - NAME = "name" - LOG_BUCKET = "logs" - LOG_PFX = "pfx" - before = {"logging": None} - bucket = self._make_one(name=NAME, properties=before) - self.assertIsNone(bucket.get_logging()) - bucket.enable_logging(LOG_BUCKET, LOG_PFX) - info = bucket.get_logging() - self.assertEqual(info["logBucket"], LOG_BUCKET) - self.assertEqual(info["logObjectPrefix"], LOG_PFX) - - def test_disable_logging(self): - NAME = "name" - before = {"logging": {"logBucket": "logs", "logObjectPrefix": "pfx"}} - bucket = self._make_one(name=NAME, properties=before) - self.assertIsNotNone(bucket.get_logging()) - bucket.disable_logging() - self.assertIsNone(bucket.get_logging()) - - def test_metageneration(self): - METAGENERATION = 42 - properties = {"metageneration": METAGENERATION} - bucket = self._make_one(properties=properties) - self.assertEqual(bucket.metageneration, METAGENERATION) - - def test_metageneration_unset(self): - bucket = self._make_one() - self.assertIsNone(bucket.metageneration) - - def test_metageneration_string_val(self): - METAGENERATION = 42 - properties = {"metageneration": str(METAGENERATION)} - bucket = self._make_one(properties=properties) - self.assertEqual(bucket.metageneration, METAGENERATION) - - def test_owner(self): - OWNER = {"entity": "project-owner-12345", "entityId": "23456"} - properties = {"owner": OWNER} - bucket = self._make_one(properties=properties) - owner = bucket.owner - self.assertEqual(owner["entity"], "project-owner-12345") - self.assertEqual(owner["entityId"], "23456") - - def test_project_number(self): - PROJECT_NUMBER = 12345 - properties = {"projectNumber": PROJECT_NUMBER} - bucket = self._make_one(properties=properties) - self.assertEqual(bucket.project_number, PROJECT_NUMBER) - - def test_project_number_unset(self): - bucket = self._make_one() - self.assertIsNone(bucket.project_number) - - def test_project_number_string_val(self): - PROJECT_NUMBER = 12345 - properties = {"projectNumber": str(PROJECT_NUMBER)} - bucket = self._make_one(properties=properties) - self.assertEqual(bucket.project_number, PROJECT_NUMBER) - - def test_retention_policy_effective_time_policy_missing(self): - bucket = self._make_one() - self.assertIsNone(bucket.retention_policy_effective_time) - - def test_retention_policy_effective_time_et_missing(self): - properties = {"retentionPolicy": {}} - bucket = self._make_one(properties=properties) - - self.assertIsNone(bucket.retention_policy_effective_time) - - def test_retention_policy_effective_time(self): - import datetime - from google.cloud._helpers import _datetime_to_rfc3339 - from google.cloud._helpers import UTC - - effective_time = datetime.datetime.utcnow().replace(tzinfo=UTC) - properties = { - "retentionPolicy": {"effectiveTime": _datetime_to_rfc3339(effective_time)} - } - bucket = self._make_one(properties=properties) - - self.assertEqual(bucket.retention_policy_effective_time, effective_time) - - def test_retention_policy_locked_missing(self): - bucket = self._make_one() - self.assertFalse(bucket.retention_policy_locked) - - def test_retention_policy_locked_false(self): - properties = {"retentionPolicy": {"isLocked": False}} - bucket = self._make_one(properties=properties) - self.assertFalse(bucket.retention_policy_locked) - - def test_retention_policy_locked_true(self): - properties = {"retentionPolicy": {"isLocked": True}} - bucket = self._make_one(properties=properties) - self.assertTrue(bucket.retention_policy_locked) - - def test_retention_period_getter_policymissing(self): - bucket = self._make_one() - - self.assertIsNone(bucket.retention_period) - - def test_retention_period_getter_pr_missing(self): - properties = {"retentionPolicy": {}} - bucket = self._make_one(properties=properties) - - self.assertIsNone(bucket.retention_period) - - def test_retention_period_getter(self): - period = 86400 * 100 # 100 days - properties = {"retentionPolicy": {"retentionPeriod": str(period)}} - bucket = self._make_one(properties=properties) - - self.assertEqual(bucket.retention_period, period) - - def test_retention_period_setter_w_none(self): - period = 86400 * 100 # 100 days - bucket = self._make_one() - bucket._properties["retentionPolicy"] = {"retentionPeriod": period} - - bucket.retention_period = None - - self.assertIsNone(bucket._properties["retentionPolicy"]) - - def test_retention_period_setter_w_int(self): - period = 86400 * 100 # 100 days - bucket = self._make_one() - - bucket.retention_period = period - - self.assertEqual( - bucket._properties["retentionPolicy"]["retentionPeriod"], str(period) - ) - - def test_self_link(self): - SELF_LINK = "http://example.com/self/" - properties = {"selfLink": SELF_LINK} - bucket = self._make_one(properties=properties) - self.assertEqual(bucket.self_link, SELF_LINK) - - def test_storage_class_getter(self): - from google.cloud.storage.constants import NEARLINE_STORAGE_CLASS - - properties = {"storageClass": NEARLINE_STORAGE_CLASS} - bucket = self._make_one(properties=properties) - self.assertEqual(bucket.storage_class, NEARLINE_STORAGE_CLASS) - - def test_storage_class_setter_invalid(self): - NAME = "name" - bucket = self._make_one(name=NAME) - with self.assertRaises(ValueError): - bucket.storage_class = "BOGUS" - self.assertFalse("storageClass" in bucket._changes) - - def test_storage_class_setter_STANDARD(self): - from google.cloud.storage.constants import STANDARD_STORAGE_CLASS - - NAME = "name" - bucket = self._make_one(name=NAME) - bucket.storage_class = STANDARD_STORAGE_CLASS - self.assertEqual(bucket.storage_class, STANDARD_STORAGE_CLASS) - self.assertTrue("storageClass" in bucket._changes) - - def test_storage_class_setter_NEARLINE(self): - from google.cloud.storage.constants import NEARLINE_STORAGE_CLASS - - NAME = "name" - bucket = self._make_one(name=NAME) - bucket.storage_class = NEARLINE_STORAGE_CLASS - self.assertEqual(bucket.storage_class, NEARLINE_STORAGE_CLASS) - self.assertTrue("storageClass" in bucket._changes) - - def test_storage_class_setter_COLDLINE(self): - from google.cloud.storage.constants import COLDLINE_STORAGE_CLASS - - NAME = "name" - bucket = self._make_one(name=NAME) - bucket.storage_class = COLDLINE_STORAGE_CLASS - self.assertEqual(bucket.storage_class, COLDLINE_STORAGE_CLASS) - self.assertTrue("storageClass" in bucket._changes) - - def test_storage_class_setter_ARCHIVE(self): - from google.cloud.storage.constants import ARCHIVE_STORAGE_CLASS - - NAME = "name" - bucket = self._make_one(name=NAME) - bucket.storage_class = ARCHIVE_STORAGE_CLASS - self.assertEqual(bucket.storage_class, ARCHIVE_STORAGE_CLASS) - self.assertTrue("storageClass" in bucket._changes) - - def test_storage_class_setter_MULTI_REGIONAL(self): - from google.cloud.storage.constants import MULTI_REGIONAL_LEGACY_STORAGE_CLASS - - NAME = "name" - bucket = self._make_one(name=NAME) - bucket.storage_class = MULTI_REGIONAL_LEGACY_STORAGE_CLASS - self.assertEqual(bucket.storage_class, MULTI_REGIONAL_LEGACY_STORAGE_CLASS) - self.assertTrue("storageClass" in bucket._changes) - - def test_storage_class_setter_REGIONAL(self): - from google.cloud.storage.constants import REGIONAL_LEGACY_STORAGE_CLASS - - NAME = "name" - bucket = self._make_one(name=NAME) - bucket.storage_class = REGIONAL_LEGACY_STORAGE_CLASS - self.assertEqual(bucket.storage_class, REGIONAL_LEGACY_STORAGE_CLASS) - self.assertTrue("storageClass" in bucket._changes) - - def test_storage_class_setter_DURABLE_REDUCED_AVAILABILITY(self): - from google.cloud.storage.constants import ( - DURABLE_REDUCED_AVAILABILITY_LEGACY_STORAGE_CLASS, - ) - - NAME = "name" - bucket = self._make_one(name=NAME) - bucket.storage_class = DURABLE_REDUCED_AVAILABILITY_LEGACY_STORAGE_CLASS - self.assertEqual( - bucket.storage_class, DURABLE_REDUCED_AVAILABILITY_LEGACY_STORAGE_CLASS - ) - self.assertTrue("storageClass" in bucket._changes) - - def test_time_created(self): - from google.cloud._helpers import _RFC3339_MICROS - from google.cloud._helpers import UTC - - TIMESTAMP = datetime.datetime(2014, 11, 5, 20, 34, 37, tzinfo=UTC) - TIME_CREATED = TIMESTAMP.strftime(_RFC3339_MICROS) - properties = {"timeCreated": TIME_CREATED} - bucket = self._make_one(properties=properties) - self.assertEqual(bucket.time_created, TIMESTAMP) - - def test_time_created_unset(self): - bucket = self._make_one() - self.assertIsNone(bucket.time_created) - - def test_versioning_enabled_getter_missing(self): - NAME = "name" - bucket = self._make_one(name=NAME) - self.assertEqual(bucket.versioning_enabled, False) - - def test_versioning_enabled_getter(self): - NAME = "name" - before = {"versioning": {"enabled": True}} - bucket = self._make_one(name=NAME, properties=before) - self.assertEqual(bucket.versioning_enabled, True) - - @mock.patch("warnings.warn") - def test_create_deprecated(self, mock_warn): - from google.cloud.storage.client import Client - - PROJECT = "PROJECT" - BUCKET_NAME = "bucket-name" - DATA = {"name": BUCKET_NAME} - connection = _make_connection(DATA) - client = Client(project=PROJECT) - client._base_connection = connection - - bucket = self._make_one(client=client, name=BUCKET_NAME) - bucket.create() - - connection.api_request.assert_called_once_with( - method="POST", - path="/b", - query_params={"project": PROJECT}, - data=DATA, - _target_object=bucket, - timeout=self._get_default_timeout(), - ) - - mock_warn.assert_called_with( - "Bucket.create() is deprecated and will be removed in future." - "Use Client.create_bucket() instead.", - PendingDeprecationWarning, - stacklevel=1, - ) - - def test_create_w_user_project(self): - from google.cloud.storage.client import Client - - PROJECT = "PROJECT" - BUCKET_NAME = "bucket-name" - DATA = {"name": BUCKET_NAME} - connection = _make_connection(DATA) - client = Client(project=PROJECT) - client._base_connection = connection - - bucket = self._make_one(client=client, name=BUCKET_NAME) - bucket._user_project = "USER_PROJECT" - with self.assertRaises(ValueError): - bucket.create() - - def test_versioning_enabled_setter(self): - NAME = "name" - bucket = self._make_one(name=NAME) - self.assertFalse(bucket.versioning_enabled) - bucket.versioning_enabled = True - self.assertTrue(bucket.versioning_enabled) - - def test_requester_pays_getter_missing(self): - NAME = "name" - bucket = self._make_one(name=NAME) - self.assertEqual(bucket.requester_pays, False) - - def test_requester_pays_getter(self): - NAME = "name" - before = {"billing": {"requesterPays": True}} - bucket = self._make_one(name=NAME, properties=before) - self.assertEqual(bucket.requester_pays, True) - - def test_requester_pays_setter(self): - NAME = "name" - bucket = self._make_one(name=NAME) - self.assertFalse(bucket.requester_pays) - bucket.requester_pays = True - self.assertTrue(bucket.requester_pays) - - def test_configure_website_defaults(self): - NAME = "name" - UNSET = {"website": {"mainPageSuffix": None, "notFoundPage": None}} - bucket = self._make_one(name=NAME) - bucket.configure_website() - self.assertEqual(bucket._properties, UNSET) - - def test_configure_website(self): - NAME = "name" - WEBSITE_VAL = { - "website": {"mainPageSuffix": "html", "notFoundPage": "404.html"} - } - bucket = self._make_one(name=NAME) - bucket.configure_website("html", "404.html") - self.assertEqual(bucket._properties, WEBSITE_VAL) - - def test_disable_website(self): - NAME = "name" - UNSET = {"website": {"mainPageSuffix": None, "notFoundPage": None}} - bucket = self._make_one(name=NAME) - bucket.disable_website() - self.assertEqual(bucket._properties, UNSET) - - def test_get_iam_policy(self): - from google.cloud.storage.iam import STORAGE_OWNER_ROLE - from google.cloud.storage.iam import STORAGE_EDITOR_ROLE - from google.cloud.storage.iam import STORAGE_VIEWER_ROLE - from google.api_core.iam import Policy - - NAME = "name" - PATH = "/b/%s" % (NAME,) - ETAG = "DEADBEEF" - VERSION = 1 - OWNER1 = "user:phred@example.com" - OWNER2 = "group:cloud-logs@google.com" - EDITOR1 = "domain:google.com" - EDITOR2 = "user:phred@example.com" - VIEWER1 = "serviceAccount:1234-abcdef@service.example.com" - VIEWER2 = "user:phred@example.com" - RETURNED = { - "resourceId": PATH, - "etag": ETAG, - "version": VERSION, - "bindings": [ - {"role": STORAGE_OWNER_ROLE, "members": [OWNER1, OWNER2]}, - {"role": STORAGE_EDITOR_ROLE, "members": [EDITOR1, EDITOR2]}, - {"role": STORAGE_VIEWER_ROLE, "members": [VIEWER1, VIEWER2]}, - ], - } - EXPECTED = { - binding["role"]: set(binding["members"]) for binding in RETURNED["bindings"] - } - connection = _Connection(RETURNED) - client = _Client(connection, None) - bucket = self._make_one(client=client, name=NAME) - - policy = bucket.get_iam_policy(timeout=42) - - self.assertIsInstance(policy, Policy) - self.assertEqual(policy.etag, RETURNED["etag"]) - self.assertEqual(policy.version, RETURNED["version"]) - self.assertEqual(dict(policy), EXPECTED) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "GET") - self.assertEqual(kw[0]["path"], "%s/iam" % (PATH,)) - self.assertEqual(kw[0]["query_params"], {}) - self.assertEqual(kw[0]["timeout"], 42) - - def test_get_iam_policy_w_user_project(self): - from google.api_core.iam import Policy - - NAME = "name" - USER_PROJECT = "user-project-123" - PATH = "/b/%s" % (NAME,) - ETAG = "DEADBEEF" - VERSION = 1 - RETURNED = { - "resourceId": PATH, - "etag": ETAG, - "version": VERSION, - "bindings": [], - } - EXPECTED = {} - connection = _Connection(RETURNED) - client = _Client(connection, None) - bucket = self._make_one(client=client, name=NAME, user_project=USER_PROJECT) - - policy = bucket.get_iam_policy() - - self.assertIsInstance(policy, Policy) - self.assertEqual(policy.etag, RETURNED["etag"]) - self.assertEqual(policy.version, RETURNED["version"]) - self.assertEqual(dict(policy), EXPECTED) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "GET") - self.assertEqual(kw[0]["path"], "%s/iam" % (PATH,)) - self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - - def test_get_iam_policy_w_requested_policy_version(self): - from google.cloud.storage.iam import STORAGE_OWNER_ROLE - - NAME = "name" - PATH = "/b/%s" % (NAME,) - ETAG = "DEADBEEF" - VERSION = 1 - OWNER1 = "user:phred@example.com" - OWNER2 = "group:cloud-logs@google.com" - RETURNED = { - "resourceId": PATH, - "etag": ETAG, - "version": VERSION, - "bindings": [{"role": STORAGE_OWNER_ROLE, "members": [OWNER1, OWNER2]}], - } - connection = _Connection(RETURNED) - client = _Client(connection, None) - bucket = self._make_one(client=client, name=NAME) - - policy = bucket.get_iam_policy(requested_policy_version=3) - - self.assertEqual(policy.version, VERSION) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "GET") - self.assertEqual(kw[0]["path"], "%s/iam" % (PATH,)) - self.assertEqual(kw[0]["query_params"], {"optionsRequestedPolicyVersion": 3}) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - - def test_set_iam_policy(self): - import operator - from google.cloud.storage.iam import STORAGE_OWNER_ROLE - from google.cloud.storage.iam import STORAGE_EDITOR_ROLE - from google.cloud.storage.iam import STORAGE_VIEWER_ROLE - from google.api_core.iam import Policy - - NAME = "name" - PATH = "/b/%s" % (NAME,) - ETAG = "DEADBEEF" - VERSION = 1 - OWNER1 = "user:phred@example.com" - OWNER2 = "group:cloud-logs@google.com" - EDITOR1 = "domain:google.com" - EDITOR2 = "user:phred@example.com" - VIEWER1 = "serviceAccount:1234-abcdef@service.example.com" - VIEWER2 = "user:phred@example.com" - BINDINGS = [ - {"role": STORAGE_OWNER_ROLE, "members": [OWNER1, OWNER2]}, - {"role": STORAGE_EDITOR_ROLE, "members": [EDITOR1, EDITOR2]}, - {"role": STORAGE_VIEWER_ROLE, "members": [VIEWER1, VIEWER2]}, - ] - RETURNED = {"etag": ETAG, "version": VERSION, "bindings": BINDINGS} - policy = Policy() - for binding in BINDINGS: - policy[binding["role"]] = binding["members"] - - connection = _Connection(RETURNED) - client = _Client(connection, None) - bucket = self._make_one(client=client, name=NAME) - - returned = bucket.set_iam_policy(policy, timeout=42) - - self.assertEqual(returned.etag, ETAG) - self.assertEqual(returned.version, VERSION) - self.assertEqual(dict(returned), dict(policy)) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "PUT") - self.assertEqual(kw[0]["path"], "%s/iam" % (PATH,)) - self.assertEqual(kw[0]["query_params"], {}) - self.assertEqual(kw[0]["timeout"], 42) - sent = kw[0]["data"] - self.assertEqual(sent["resourceId"], PATH) - self.assertEqual(len(sent["bindings"]), len(BINDINGS)) - key = operator.itemgetter("role") - for found, expected in zip( - sorted(sent["bindings"], key=key), sorted(BINDINGS, key=key) - ): - self.assertEqual(found["role"], expected["role"]) - self.assertEqual(sorted(found["members"]), sorted(expected["members"])) - - def test_set_iam_policy_w_user_project(self): - import operator - from google.cloud.storage.iam import STORAGE_OWNER_ROLE - from google.cloud.storage.iam import STORAGE_EDITOR_ROLE - from google.cloud.storage.iam import STORAGE_VIEWER_ROLE - from google.api_core.iam import Policy - - NAME = "name" - USER_PROJECT = "user-project-123" - PATH = "/b/%s" % (NAME,) - ETAG = "DEADBEEF" - VERSION = 1 - OWNER1 = "user:phred@example.com" - OWNER2 = "group:cloud-logs@google.com" - EDITOR1 = "domain:google.com" - EDITOR2 = "user:phred@example.com" - VIEWER1 = "serviceAccount:1234-abcdef@service.example.com" - VIEWER2 = "user:phred@example.com" - BINDINGS = [ - {"role": STORAGE_OWNER_ROLE, "members": [OWNER1, OWNER2]}, - {"role": STORAGE_EDITOR_ROLE, "members": [EDITOR1, EDITOR2]}, - {"role": STORAGE_VIEWER_ROLE, "members": [VIEWER1, VIEWER2]}, - ] - RETURNED = {"etag": ETAG, "version": VERSION, "bindings": BINDINGS} - policy = Policy() - for binding in BINDINGS: - policy[binding["role"]] = binding["members"] - - connection = _Connection(RETURNED) - client = _Client(connection, None) - bucket = self._make_one(client=client, name=NAME, user_project=USER_PROJECT) - - returned = bucket.set_iam_policy(policy) - - self.assertEqual(returned.etag, ETAG) - self.assertEqual(returned.version, VERSION) - self.assertEqual(dict(returned), dict(policy)) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "PUT") - self.assertEqual(kw[0]["path"], "%s/iam" % (PATH,)) - self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - sent = kw[0]["data"] - self.assertEqual(sent["resourceId"], PATH) - self.assertEqual(len(sent["bindings"]), len(BINDINGS)) - key = operator.itemgetter("role") - for found, expected in zip( - sorted(sent["bindings"], key=key), sorted(BINDINGS, key=key) - ): - self.assertEqual(found["role"], expected["role"]) - self.assertEqual(sorted(found["members"]), sorted(expected["members"])) - - def test_test_iam_permissions(self): - from google.cloud.storage.iam import STORAGE_OBJECTS_LIST - from google.cloud.storage.iam import STORAGE_BUCKETS_GET - from google.cloud.storage.iam import STORAGE_BUCKETS_UPDATE - - NAME = "name" - PATH = "/b/%s" % (NAME,) - PERMISSIONS = [ - STORAGE_OBJECTS_LIST, - STORAGE_BUCKETS_GET, - STORAGE_BUCKETS_UPDATE, - ] - ALLOWED = PERMISSIONS[1:] - RETURNED = {"permissions": ALLOWED} - connection = _Connection(RETURNED) - client = _Client(connection, None) - bucket = self._make_one(client=client, name=NAME) - - allowed = bucket.test_iam_permissions(PERMISSIONS, timeout=42) - - self.assertEqual(allowed, ALLOWED) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "GET") - self.assertEqual(kw[0]["path"], "%s/iam/testPermissions" % (PATH,)) - self.assertEqual(kw[0]["query_params"], {"permissions": PERMISSIONS}) - self.assertEqual(kw[0]["timeout"], 42) - - def test_test_iam_permissions_w_user_project(self): - from google.cloud.storage.iam import STORAGE_OBJECTS_LIST - from google.cloud.storage.iam import STORAGE_BUCKETS_GET - from google.cloud.storage.iam import STORAGE_BUCKETS_UPDATE - - NAME = "name" - USER_PROJECT = "user-project-123" - PATH = "/b/%s" % (NAME,) - PERMISSIONS = [ - STORAGE_OBJECTS_LIST, - STORAGE_BUCKETS_GET, - STORAGE_BUCKETS_UPDATE, - ] - ALLOWED = PERMISSIONS[1:] - RETURNED = {"permissions": ALLOWED} - connection = _Connection(RETURNED) - client = _Client(connection, None) - bucket = self._make_one(client=client, name=NAME, user_project=USER_PROJECT) - - allowed = bucket.test_iam_permissions(PERMISSIONS) - - self.assertEqual(allowed, ALLOWED) - - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "GET") - self.assertEqual(kw[0]["path"], "%s/iam/testPermissions" % (PATH,)) - self.assertEqual( - kw[0]["query_params"], - {"permissions": PERMISSIONS, "userProject": USER_PROJECT}, - ) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - - def test_make_public_defaults(self): - from google.cloud.storage.acl import _ACLEntity - - NAME = "name" - permissive = [{"entity": "allUsers", "role": _ACLEntity.READER_ROLE}] - after = {"acl": permissive, "defaultObjectAcl": []} - connection = _Connection(after) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - bucket.acl.loaded = True - bucket.default_object_acl.loaded = True - bucket.make_public() - self.assertEqual(list(bucket.acl), permissive) - self.assertEqual(list(bucket.default_object_acl), []) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "PATCH") - self.assertEqual(kw[0]["path"], "/b/%s" % NAME) - self.assertEqual(kw[0]["data"], {"acl": after["acl"]}) - self.assertEqual(kw[0]["query_params"], {"projection": "full"}) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - - def _make_public_w_future_helper(self, default_object_acl_loaded=True): - from google.cloud.storage.acl import _ACLEntity - - NAME = "name" - permissive = [{"entity": "allUsers", "role": _ACLEntity.READER_ROLE}] - after1 = {"acl": permissive, "defaultObjectAcl": []} - after2 = {"acl": permissive, "defaultObjectAcl": permissive} - if default_object_acl_loaded: - num_requests = 2 - connection = _Connection(after1, after2) - else: - num_requests = 3 - # We return the same value for default_object_acl.reload() - # to consume. - connection = _Connection(after1, after1, after2) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - bucket.acl.loaded = True - bucket.default_object_acl.loaded = default_object_acl_loaded - bucket.make_public(future=True) - self.assertEqual(list(bucket.acl), permissive) - self.assertEqual(list(bucket.default_object_acl), permissive) - kw = connection._requested - self.assertEqual(len(kw), num_requests) - self.assertEqual(kw[0]["method"], "PATCH") - self.assertEqual(kw[0]["path"], "/b/%s" % NAME) - self.assertEqual(kw[0]["data"], {"acl": permissive}) - self.assertEqual(kw[0]["query_params"], {"projection": "full"}) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - if not default_object_acl_loaded: - self.assertEqual(kw[1]["method"], "GET") - self.assertEqual(kw[1]["path"], "/b/%s/defaultObjectAcl" % NAME) - # Last could be 1 or 2 depending on `default_object_acl_loaded`. - self.assertEqual(kw[-1]["method"], "PATCH") - self.assertEqual(kw[-1]["path"], "/b/%s" % NAME) - self.assertEqual(kw[-1]["data"], {"defaultObjectAcl": permissive}) - self.assertEqual(kw[-1]["query_params"], {"projection": "full"}) - self.assertEqual(kw[-1]["timeout"], self._get_default_timeout()) - - def test_make_public_w_future(self): - self._make_public_w_future_helper(default_object_acl_loaded=True) - - def test_make_public_w_future_reload_default(self): - self._make_public_w_future_helper(default_object_acl_loaded=False) - - def test_make_public_recursive(self): - from google.cloud.storage.acl import _ACLEntity - - _saved = [] - - class _Blob(object): - _granted = False - - def __init__(self, bucket, name): - self._bucket = bucket - self._name = name - - @property - def acl(self): - return self - - # Faux ACL methods - def all(self): - return self - - def grant_read(self): - self._granted = True - - def save(self, client=None, timeout=None): - _saved.append( - (self._bucket, self._name, self._granted, client, timeout) - ) - - def item_to_blob(self, item): - return _Blob(self.bucket, item["name"]) - - NAME = "name" - BLOB_NAME = "blob-name" - permissive = [{"entity": "allUsers", "role": _ACLEntity.READER_ROLE}] - after = {"acl": permissive, "defaultObjectAcl": []} - connection = _Connection(after, {"items": [{"name": BLOB_NAME}]}) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - bucket.acl.loaded = True - bucket.default_object_acl.loaded = True - - with mock.patch("google.cloud.storage.bucket._item_to_blob", new=item_to_blob): - bucket.make_public(recursive=True, timeout=42) - self.assertEqual(list(bucket.acl), permissive) - self.assertEqual(list(bucket.default_object_acl), []) - self.assertEqual(_saved, [(bucket, BLOB_NAME, True, None, 42)]) - kw = connection._requested - self.assertEqual(len(kw), 2) - self.assertEqual(kw[0]["method"], "PATCH") - self.assertEqual(kw[0]["path"], "/b/%s" % NAME) - self.assertEqual(kw[0]["data"], {"acl": permissive}) - self.assertEqual(kw[0]["query_params"], {"projection": "full"}) - self.assertEqual(kw[0]["timeout"], 42) - self.assertEqual(kw[1]["method"], "GET") - self.assertEqual(kw[1]["path"], "/b/%s/o" % NAME) - max_results = bucket._MAX_OBJECTS_FOR_ITERATION + 1 - self.assertEqual( - kw[1]["query_params"], {"maxResults": max_results, "projection": "full"} - ) - self.assertEqual(kw[1]["timeout"], 42) - - def test_make_public_recursive_too_many(self): - from google.cloud.storage.acl import _ACLEntity - - PERMISSIVE = [{"entity": "allUsers", "role": _ACLEntity.READER_ROLE}] - AFTER = {"acl": PERMISSIVE, "defaultObjectAcl": []} - - NAME = "name" - BLOB_NAME1 = "blob-name1" - BLOB_NAME2 = "blob-name2" - GET_BLOBS_RESP = {"items": [{"name": BLOB_NAME1}, {"name": BLOB_NAME2}]} - connection = _Connection(AFTER, GET_BLOBS_RESP) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - bucket.acl.loaded = True - bucket.default_object_acl.loaded = True - - # Make the Bucket refuse to make_public with 2 objects. - bucket._MAX_OBJECTS_FOR_ITERATION = 1 - self.assertRaises(ValueError, bucket.make_public, recursive=True) - - def test_make_private_defaults(self): - NAME = "name" - no_permissions = [] - after = {"acl": no_permissions, "defaultObjectAcl": []} - connection = _Connection(after) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - bucket.acl.loaded = True - bucket.default_object_acl.loaded = True - bucket.make_private() - self.assertEqual(list(bucket.acl), no_permissions) - self.assertEqual(list(bucket.default_object_acl), []) - kw = connection._requested - self.assertEqual(len(kw), 1) - self.assertEqual(kw[0]["method"], "PATCH") - self.assertEqual(kw[0]["path"], "/b/%s" % NAME) - self.assertEqual(kw[0]["data"], {"acl": after["acl"]}) - self.assertEqual(kw[0]["query_params"], {"projection": "full"}) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - - def _make_private_w_future_helper(self, default_object_acl_loaded=True): - NAME = "name" - no_permissions = [] - after1 = {"acl": no_permissions, "defaultObjectAcl": []} - after2 = {"acl": no_permissions, "defaultObjectAcl": no_permissions} - if default_object_acl_loaded: - num_requests = 2 - connection = _Connection(after1, after2) - else: - num_requests = 3 - # We return the same value for default_object_acl.reload() - # to consume. - connection = _Connection(after1, after1, after2) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - bucket.acl.loaded = True - bucket.default_object_acl.loaded = default_object_acl_loaded - bucket.make_private(future=True) - self.assertEqual(list(bucket.acl), no_permissions) - self.assertEqual(list(bucket.default_object_acl), no_permissions) - kw = connection._requested - self.assertEqual(len(kw), num_requests) - self.assertEqual(kw[0]["method"], "PATCH") - self.assertEqual(kw[0]["path"], "/b/%s" % NAME) - self.assertEqual(kw[0]["data"], {"acl": no_permissions}) - self.assertEqual(kw[0]["query_params"], {"projection": "full"}) - self.assertEqual(kw[0]["timeout"], self._get_default_timeout()) - if not default_object_acl_loaded: - self.assertEqual(kw[1]["method"], "GET") - self.assertEqual(kw[1]["path"], "/b/%s/defaultObjectAcl" % NAME) - # Last could be 1 or 2 depending on `default_object_acl_loaded`. - self.assertEqual(kw[-1]["method"], "PATCH") - self.assertEqual(kw[-1]["path"], "/b/%s" % NAME) - self.assertEqual(kw[-1]["data"], {"defaultObjectAcl": no_permissions}) - self.assertEqual(kw[-1]["query_params"], {"projection": "full"}) - self.assertEqual(kw[-1]["timeout"], self._get_default_timeout()) - - def test_make_private_w_future(self): - self._make_private_w_future_helper(default_object_acl_loaded=True) - - def test_make_private_w_future_reload_default(self): - self._make_private_w_future_helper(default_object_acl_loaded=False) - - def test_make_private_recursive(self): - _saved = [] - - class _Blob(object): - _granted = True - - def __init__(self, bucket, name): - self._bucket = bucket - self._name = name - - @property - def acl(self): - return self - - # Faux ACL methods - def all(self): - return self - - def revoke_read(self): - self._granted = False - - def save(self, client=None, timeout=None): - _saved.append( - (self._bucket, self._name, self._granted, client, timeout) - ) - - def item_to_blob(self, item): - return _Blob(self.bucket, item["name"]) - - NAME = "name" - BLOB_NAME = "blob-name" - no_permissions = [] - after = {"acl": no_permissions, "defaultObjectAcl": []} - connection = _Connection(after, {"items": [{"name": BLOB_NAME}]}) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - bucket.acl.loaded = True - bucket.default_object_acl.loaded = True - - with mock.patch("google.cloud.storage.bucket._item_to_blob", new=item_to_blob): - bucket.make_private(recursive=True, timeout=42) - self.assertEqual(list(bucket.acl), no_permissions) - self.assertEqual(list(bucket.default_object_acl), []) - self.assertEqual(_saved, [(bucket, BLOB_NAME, False, None, 42)]) - kw = connection._requested - self.assertEqual(len(kw), 2) - self.assertEqual(kw[0]["method"], "PATCH") - self.assertEqual(kw[0]["path"], "/b/%s" % NAME) - self.assertEqual(kw[0]["data"], {"acl": no_permissions}) - self.assertEqual(kw[0]["query_params"], {"projection": "full"}) - self.assertEqual(kw[0]["timeout"], 42) - self.assertEqual(kw[1]["method"], "GET") - self.assertEqual(kw[1]["path"], "/b/%s/o" % NAME) - max_results = bucket._MAX_OBJECTS_FOR_ITERATION + 1 - self.assertEqual( - kw[1]["query_params"], {"maxResults": max_results, "projection": "full"} - ) - self.assertEqual(kw[1]["timeout"], 42) - - def test_make_private_recursive_too_many(self): - NO_PERMISSIONS = [] - AFTER = {"acl": NO_PERMISSIONS, "defaultObjectAcl": []} - - NAME = "name" - BLOB_NAME1 = "blob-name1" - BLOB_NAME2 = "blob-name2" - GET_BLOBS_RESP = {"items": [{"name": BLOB_NAME1}, {"name": BLOB_NAME2}]} - connection = _Connection(AFTER, GET_BLOBS_RESP) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - bucket.acl.loaded = True - bucket.default_object_acl.loaded = True - - # Make the Bucket refuse to make_private with 2 objects. - bucket._MAX_OBJECTS_FOR_ITERATION = 1 - self.assertRaises(ValueError, bucket.make_private, recursive=True) - - def test_page_empty_response(self): - from google.api_core import page_iterator - - connection = _Connection() - client = _Client(connection) - name = "name" - bucket = self._make_one(client=client, name=name) - iterator = bucket.list_blobs() - page = page_iterator.Page(iterator, (), None) - iterator._page = page - blobs = list(page) - self.assertEqual(blobs, []) - self.assertEqual(iterator.prefixes, set()) - - def test_page_non_empty_response(self): - import six - from google.cloud.storage.blob import Blob - - blob_name = "blob-name" - response = {"items": [{"name": blob_name}], "prefixes": ["foo"]} - connection = _Connection() - client = _Client(connection) - name = "name" - bucket = self._make_one(client=client, name=name) - - def dummy_response(): - return response - - iterator = bucket.list_blobs() - iterator._get_next_page_response = dummy_response - - page = six.next(iterator.pages) - self.assertEqual(page.prefixes, ("foo",)) - self.assertEqual(page.num_items, 1) - blob = six.next(page) - self.assertEqual(page.remaining, 0) - self.assertIsInstance(blob, Blob) - self.assertEqual(blob.name, blob_name) - self.assertEqual(iterator.prefixes, set(["foo"])) - - def test_cumulative_prefixes(self): - import six - from google.cloud.storage.blob import Blob - - BLOB_NAME = "blob-name1" - response1 = { - "items": [{"name": BLOB_NAME}], - "prefixes": ["foo"], - "nextPageToken": "s39rmf9", - } - response2 = {"items": [], "prefixes": ["bar"]} - connection = _Connection() - client = _Client(connection) - name = "name" - bucket = self._make_one(client=client, name=name) - responses = [response1, response2] - - def dummy_response(): - return responses.pop(0) - - iterator = bucket.list_blobs() - iterator._get_next_page_response = dummy_response - - # Parse first response. - pages_iter = iterator.pages - page1 = six.next(pages_iter) - self.assertEqual(page1.prefixes, ("foo",)) - self.assertEqual(page1.num_items, 1) - blob = six.next(page1) - self.assertEqual(page1.remaining, 0) - self.assertIsInstance(blob, Blob) - self.assertEqual(blob.name, BLOB_NAME) - self.assertEqual(iterator.prefixes, set(["foo"])) - # Parse second response. - page2 = six.next(pages_iter) - self.assertEqual(page2.prefixes, ("bar",)) - self.assertEqual(page2.num_items, 0) - self.assertEqual(iterator.prefixes, set(["foo", "bar"])) - - def _test_generate_upload_policy_helper(self, **kwargs): - import base64 - import json - - credentials = _create_signing_credentials() - credentials.signer_email = mock.sentinel.signer_email - credentials.sign_bytes.return_value = b"DEADBEEF" - connection = _Connection() - connection.credentials = credentials - client = _Client(connection) - name = "name" - bucket = self._make_one(client=client, name=name) - - conditions = [["starts-with", "$key", ""]] - - policy_fields = bucket.generate_upload_policy(conditions, **kwargs) - - self.assertEqual(policy_fields["bucket"], bucket.name) - self.assertEqual(policy_fields["GoogleAccessId"], mock.sentinel.signer_email) - self.assertEqual( - policy_fields["signature"], base64.b64encode(b"DEADBEEF").decode("utf-8") - ) - - policy = json.loads(base64.b64decode(policy_fields["policy"]).decode("utf-8")) - - policy_conditions = policy["conditions"] - expected_conditions = [{"bucket": bucket.name}] + conditions - for expected_condition in expected_conditions: - for condition in policy_conditions: - if condition == expected_condition: - break - else: # pragma: NO COVER - self.fail( - "Condition {} not found in {}".format( - expected_condition, policy_conditions - ) - ) - - return policy_fields, policy - - @mock.patch( - "google.cloud.storage.bucket._NOW", return_value=datetime.datetime(1990, 1, 1) - ) - def test_generate_upload_policy(self, now): - from google.cloud._helpers import _datetime_to_rfc3339 - - _, policy = self._test_generate_upload_policy_helper() - - self.assertEqual( - policy["expiration"], - _datetime_to_rfc3339(now() + datetime.timedelta(hours=1)), - ) - - def test_generate_upload_policy_args(self): - from google.cloud._helpers import _datetime_to_rfc3339 - - expiration = datetime.datetime(1990, 5, 29) - - _, policy = self._test_generate_upload_policy_helper(expiration=expiration) - - self.assertEqual(policy["expiration"], _datetime_to_rfc3339(expiration)) - - def test_generate_upload_policy_bad_credentials(self): - credentials = object() - connection = _Connection() - connection.credentials = credentials - client = _Client(connection) - name = "name" - bucket = self._make_one(client=client, name=name) - - with self.assertRaises(AttributeError): - bucket.generate_upload_policy([]) - - def test_lock_retention_policy_no_policy_set(self): - credentials = object() - connection = _Connection() - connection.credentials = credentials - client = _Client(connection) - name = "name" - bucket = self._make_one(client=client, name=name) - bucket._properties["metageneration"] = 1234 - - with self.assertRaises(ValueError): - bucket.lock_retention_policy() - - def test_lock_retention_policy_no_metageneration(self): - credentials = object() - connection = _Connection() - connection.credentials = credentials - client = _Client(connection) - name = "name" - bucket = self._make_one(client=client, name=name) - bucket._properties["retentionPolicy"] = { - "effectiveTime": "2018-03-01T16:46:27.123456Z", - "retentionPeriod": 86400 * 100, # 100 days - } - - with self.assertRaises(ValueError): - bucket.lock_retention_policy() - - def test_lock_retention_policy_already_locked(self): - credentials = object() - connection = _Connection() - connection.credentials = credentials - client = _Client(connection) - name = "name" - bucket = self._make_one(client=client, name=name) - bucket._properties["metageneration"] = 1234 - bucket._properties["retentionPolicy"] = { - "effectiveTime": "2018-03-01T16:46:27.123456Z", - "isLocked": True, - "retentionPeriod": 86400 * 100, # 100 days - } - - with self.assertRaises(ValueError): - bucket.lock_retention_policy() - - def test_lock_retention_policy_ok(self): - name = "name" - response = { - "name": name, - "metageneration": 1235, - "retentionPolicy": { - "effectiveTime": "2018-03-01T16:46:27.123456Z", - "isLocked": True, - "retentionPeriod": 86400 * 100, # 100 days - }, - } - credentials = object() - connection = _Connection(response) - connection.credentials = credentials - client = _Client(connection) - bucket = self._make_one(client=client, name=name) - bucket._properties["metageneration"] = 1234 - bucket._properties["retentionPolicy"] = { - "effectiveTime": "2018-03-01T16:46:27.123456Z", - "retentionPeriod": 86400 * 100, # 100 days - } - - bucket.lock_retention_policy(timeout=42) - - (kw,) = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], "/b/{}/lockRetentionPolicy".format(name)) - self.assertEqual(kw["query_params"], {"ifMetagenerationMatch": 1234}) - self.assertEqual(kw["timeout"], 42) - - def test_lock_retention_policy_w_user_project(self): - name = "name" - user_project = "user-project-123" - response = { - "name": name, - "metageneration": 1235, - "retentionPolicy": { - "effectiveTime": "2018-03-01T16:46:27.123456Z", - "isLocked": True, - "retentionPeriod": 86400 * 100, # 100 days - }, - } - credentials = object() - connection = _Connection(response) - connection.credentials = credentials - client = _Client(connection) - bucket = self._make_one(client=client, name=name, user_project=user_project) - bucket._properties["metageneration"] = 1234 - bucket._properties["retentionPolicy"] = { - "effectiveTime": "2018-03-01T16:46:27.123456Z", - "retentionPeriod": 86400 * 100, # 100 days - } - - bucket.lock_retention_policy() - - (kw,) = connection._requested - self.assertEqual(kw["method"], "POST") - self.assertEqual(kw["path"], "/b/{}/lockRetentionPolicy".format(name)) - self.assertEqual( - kw["query_params"], - {"ifMetagenerationMatch": 1234, "userProject": user_project}, - ) - self.assertEqual(kw["timeout"], self._get_default_timeout()) - - def test_generate_signed_url_w_invalid_version(self): - expiration = "2014-10-16T20:34:37.000Z" - connection = _Connection() - client = _Client(connection) - bucket = self._make_one(name="bucket_name", client=client) - with self.assertRaises(ValueError): - bucket.generate_signed_url(expiration, version="nonesuch") - - def _generate_signed_url_helper( - self, - version=None, - bucket_name="bucket-name", - api_access_endpoint=None, - method="GET", - content_md5=None, - content_type=None, - response_type=None, - response_disposition=None, - generation=None, - headers=None, - query_parameters=None, - credentials=None, - expiration=None, - virtual_hosted_style=False, - bucket_bound_hostname=None, - scheme="http", - ): - from six.moves.urllib import parse - from google.cloud._helpers import UTC - from google.cloud.storage.blob import _API_ACCESS_ENDPOINT - - api_access_endpoint = api_access_endpoint or _API_ACCESS_ENDPOINT - - delta = datetime.timedelta(hours=1) - - if expiration is None: - expiration = datetime.datetime.utcnow().replace(tzinfo=UTC) + delta - - connection = _Connection() - client = _Client(connection) - bucket = self._make_one(name=bucket_name, client=client) - - if version is None: - effective_version = "v2" - else: - effective_version = version - - to_patch = "google.cloud.storage.bucket.generate_signed_url_{}".format( - effective_version - ) - - with mock.patch(to_patch) as signer: - signed_uri = bucket.generate_signed_url( - expiration=expiration, - api_access_endpoint=api_access_endpoint, - method=method, - credentials=credentials, - headers=headers, - query_parameters=query_parameters, - version=version, - virtual_hosted_style=virtual_hosted_style, - bucket_bound_hostname=bucket_bound_hostname, - ) - - self.assertEqual(signed_uri, signer.return_value) - - if credentials is None: - expected_creds = client._credentials - else: - expected_creds = credentials - - if virtual_hosted_style: - expected_api_access_endpoint = "https://{}.storage.googleapis.com".format( - bucket_name - ) - elif bucket_bound_hostname: - if ":" in bucket_bound_hostname: - expected_api_access_endpoint = bucket_bound_hostname - else: - expected_api_access_endpoint = "{scheme}://{bucket_bound_hostname}".format( - scheme=scheme, bucket_bound_hostname=bucket_bound_hostname - ) - else: - expected_api_access_endpoint = api_access_endpoint - expected_resource = "/{}".format(parse.quote(bucket_name)) - - if virtual_hosted_style or bucket_bound_hostname: - expected_resource = "/" - - expected_kwargs = { - "resource": expected_resource, - "expiration": expiration, - "api_access_endpoint": expected_api_access_endpoint, - "method": method.upper(), - "headers": headers, - "query_parameters": query_parameters, - } - signer.assert_called_once_with(expected_creds, **expected_kwargs) - - def test_get_bucket_from_string_w_valid_uri(self): - from google.cloud.storage.bucket import Bucket - - connection = _Connection() - client = _Client(connection) - BUCKET_NAME = "BUCKET_NAME" - uri = "gs://" + BUCKET_NAME - bucket = Bucket.from_string(uri, client) - self.assertIsInstance(bucket, Bucket) - self.assertIs(bucket.client, client) - self.assertEqual(bucket.name, BUCKET_NAME) - - def test_get_bucket_from_string_w_invalid_uri(self): - from google.cloud.storage.bucket import Bucket - - connection = _Connection() - client = _Client(connection) - - with pytest.raises(ValueError, match="URI scheme must be gs"): - Bucket.from_string("http://bucket_name", client) - - def test_get_bucket_from_string_w_domain_name_bucket(self): - from google.cloud.storage.bucket import Bucket - - connection = _Connection() - client = _Client(connection) - BUCKET_NAME = "buckets.example.com" - uri = "gs://" + BUCKET_NAME - bucket = Bucket.from_string(uri, client) - self.assertIsInstance(bucket, Bucket) - self.assertIs(bucket.client, client) - self.assertEqual(bucket.name, BUCKET_NAME) - - def test_generate_signed_url_no_version_passed_warning(self): - self._generate_signed_url_helper() - - def _generate_signed_url_v2_helper(self, **kw): - version = "v2" - self._generate_signed_url_helper(version, **kw) - - def test_generate_signed_url_v2_w_defaults(self): - self._generate_signed_url_v2_helper() - - def test_generate_signed_url_v2_w_expiration(self): - from google.cloud._helpers import UTC - - expiration = datetime.datetime.utcnow().replace(tzinfo=UTC) - self._generate_signed_url_v2_helper(expiration=expiration) - - def test_generate_signed_url_v2_w_endpoint(self): - self._generate_signed_url_v2_helper( - api_access_endpoint="https://api.example.com/v1" - ) - - def test_generate_signed_url_v2_w_method(self): - self._generate_signed_url_v2_helper(method="POST") - - def test_generate_signed_url_v2_w_lowercase_method(self): - self._generate_signed_url_v2_helper(method="get") - - def test_generate_signed_url_v2_w_content_md5(self): - self._generate_signed_url_v2_helper(content_md5="FACEDACE") - - def test_generate_signed_url_v2_w_content_type(self): - self._generate_signed_url_v2_helper(content_type="text.html") - - def test_generate_signed_url_v2_w_response_type(self): - self._generate_signed_url_v2_helper(response_type="text.html") - - def test_generate_signed_url_v2_w_response_disposition(self): - self._generate_signed_url_v2_helper(response_disposition="inline") - - def test_generate_signed_url_v2_w_generation(self): - self._generate_signed_url_v2_helper(generation=12345) - - def test_generate_signed_url_v2_w_headers(self): - self._generate_signed_url_v2_helper(headers={"x-goog-foo": "bar"}) - - def test_generate_signed_url_v2_w_credentials(self): - credentials = object() - self._generate_signed_url_v2_helper(credentials=credentials) - - def _generate_signed_url_v4_helper(self, **kw): - version = "v4" - self._generate_signed_url_helper(version, **kw) - - def test_generate_signed_url_v4_w_defaults(self): - self._generate_signed_url_v2_helper() - - def test_generate_signed_url_v4_w_endpoint(self): - self._generate_signed_url_v4_helper( - api_access_endpoint="https://api.example.com/v1" - ) - - def test_generate_signed_url_v4_w_method(self): - self._generate_signed_url_v4_helper(method="POST") - - def test_generate_signed_url_v4_w_lowercase_method(self): - self._generate_signed_url_v4_helper(method="get") - - def test_generate_signed_url_v4_w_content_md5(self): - self._generate_signed_url_v4_helper(content_md5="FACEDACE") - - def test_generate_signed_url_v4_w_content_type(self): - self._generate_signed_url_v4_helper(content_type="text.html") - - def test_generate_signed_url_v4_w_response_type(self): - self._generate_signed_url_v4_helper(response_type="text.html") - - def test_generate_signed_url_v4_w_response_disposition(self): - self._generate_signed_url_v4_helper(response_disposition="inline") - - def test_generate_signed_url_v4_w_generation(self): - self._generate_signed_url_v4_helper(generation=12345) - - def test_generate_signed_url_v4_w_headers(self): - self._generate_signed_url_v4_helper(headers={"x-goog-foo": "bar"}) - - def test_generate_signed_url_v4_w_credentials(self): - credentials = object() - self._generate_signed_url_v4_helper(credentials=credentials) - - def test_generate_signed_url_v4_w_virtual_hostname(self): - self._generate_signed_url_v4_helper(virtual_hosted_style=True) - - def test_generate_signed_url_v4_w_bucket_bound_hostname_w_scheme(self): - self._generate_signed_url_v4_helper( - bucket_bound_hostname="http://cdn.example.com" - ) - - def test_generate_signed_url_v4_w_bucket_bound_hostname_w_bare_hostname(self): - self._generate_signed_url_v4_helper(bucket_bound_hostname="cdn.example.com") - - -class _Connection(object): - _delete_bucket = False - - def __init__(self, *responses): - self._responses = responses - self._requested = [] - self._deleted_buckets = [] - self.credentials = None - - @staticmethod - def _is_bucket_path(path): - # Now just ensure the path only has /b/ and one more segment. - return path.startswith("/b/") and path.count("/") == 2 - - def api_request(self, **kw): - from google.cloud.exceptions import NotFound - - self._requested.append(kw) - - method = kw.get("method") - path = kw.get("path", "") - if method == "DELETE" and self._is_bucket_path(path): - self._deleted_buckets.append(kw) - if self._delete_bucket: - return - else: - raise NotFound("miss") - - try: - response, self._responses = self._responses[0], self._responses[1:] - except IndexError: - raise NotFound("miss") - else: - return response - - -class _Client(object): - def __init__(self, connection, project=None): - self._base_connection = connection - self.project = project - - @property - def _connection(self): - return self._base_connection - - @property - def _credentials(self): - return self._base_connection.credentials diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 4ba98f82e..d9da3a8a8 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1471,3 +1471,178 @@ def test_get_hmac_key_metadata_w_project(self): headers=mock.ANY, timeout=self._get_default_timeout(), ) + + def test_generate_signed_post_policy(self): + import datetime + + BUCKET_NAME = "bucket-name" + BLOB_NAME = "object-name" + TIMESTAMP = "20200312T114716Z" + + credentials = _make_credentials() + credentials.sign_bytes = mock.Mock(return_value=b"Signature_bytes") + credentials.signer_email = "test@mail.com" + + client = self._make_one(project="PROJECT", credentials=credentials) + + with mock.patch( + "google.cloud.storage.client.get_v4_dtstamps", + return_value=(TIMESTAMP, "20200312"), + ): + policy = client.generate_signed_post_policy( + BUCKET_NAME, + BLOB_NAME, + conditions=[ + {"bucket": BUCKET_NAME}, + {"acl": "private"}, + ["starts-with", "$Content-Type", "text/plain"], + ], + expiration=datetime.datetime(2020, 3, 12), + ) + self.assertEqual(policy["url"], "https://storage.googleapis.com/" + BUCKET_NAME) + self.assertEqual(policy["fields"]["key"], BLOB_NAME) + self.assertEqual(policy["fields"]["x-goog-algorithm"], "GOOG4-HMAC-SHA256") + self.assertEqual(policy["fields"]["x-goog-date"], TIMESTAMP) + self.assertEqual( + policy["fields"]["x-goog-credential"], + "test@mail.com/20200312/auto/service/goog4_request", + ) + self.assertEqual( + policy["fields"]["x-goog-signature"], "5369676e61747572655f6279746573" + ) + self.assertEqual( + policy["fields"]["policy"], + b"eyJjb25kaXRpb25zIjogW3siYnVja2V0IjogImJ1Y2tldC1uYW1lIn0sIHsiYWNsIjogInByaXZhdGUifSwgWyJzdGFydHMtd2l0aCIsICIkQ29udGVudC1UeXBlIiwgInRleHQvcGxhaW4iXV0sICJleHBpcmF0aW9uIjogIjIwMjAtMDMtMTJUMDA6MDA6MDAifQ==", + ) + + def test_generate_signed_post_policy_with_fields(self): + import datetime + + BUCKET_NAME = "bucket-name" + BLOB_NAME = "object-name" + TIMESTAMP = "20200312T114716Z" + FIELD1_VALUE = "Value1" + + credentials = _make_credentials() + credentials.sign_bytes = mock.Mock(return_value=b"Signature_bytes") + credentials.signer_email = "test@mail.com" + + client = self._make_one(project="PROJECT", credentials=credentials) + + with mock.patch( + "google.cloud.storage.client.get_v4_dtstamps", + return_value=(TIMESTAMP, "20200312"), + ): + policy = client.generate_signed_post_policy( + BUCKET_NAME, + BLOB_NAME, + conditions=[ + {"bucket": BUCKET_NAME}, + {"acl": "private"}, + ["starts-with", "$Content-Type", "text/plain"], + ], + expiration=datetime.datetime(2020, 3, 12), + fields={"field1": FIELD1_VALUE, "x-ignore-field": "Ignored_value"}, + ) + self.assertEqual(policy["url"], "https://storage.googleapis.com/" + BUCKET_NAME) + self.assertEqual(policy["fields"]["key"], BLOB_NAME) + self.assertEqual(policy["fields"]["x-goog-algorithm"], "GOOG4-HMAC-SHA256") + self.assertEqual(policy["fields"]["x-goog-date"], TIMESTAMP) + self.assertEqual(policy["fields"]["field1"], FIELD1_VALUE) + self.assertNotIn("x-ignore-field", policy["fields"].keys()) + self.assertEqual( + policy["fields"]["x-goog-credential"], + "test@mail.com/20200312/auto/service/goog4_request", + ) + self.assertEqual( + policy["fields"]["x-goog-signature"], "5369676e61747572655f6279746573" + ) + self.assertEqual( + policy["fields"]["policy"], + b"eyJjb25kaXRpb25zIjogW3siYnVja2V0IjogImJ1Y2tldC1uYW1lIn0sIHsiYWNsIjogInByaXZhdGUifSwgWyJzdGFydHMtd2l0aCIsICIkQ29udGVudC1UeXBlIiwgInRleHQvcGxhaW4iXV0sICJleHBpcmF0aW9uIjogIjIwMjAtMDMtMTJUMDA6MDA6MDAifQ==", + ) + + def test_generate_signed_post_policy_virtual_hosted_style(self): + import datetime + + BUCKET_NAME = "bucket-name" + BLOB_NAME = "object-name" + TIMESTAMP = "20200312T114716Z" + + credentials = _make_credentials() + credentials.sign_bytes = mock.Mock(return_value=b"Signature_bytes") + credentials.signer_email = "test@mail.com" + + client = self._make_one(project="PROJECT", credentials=credentials) + + with mock.patch( + "google.cloud.storage.client.get_v4_dtstamps", + return_value=(TIMESTAMP, "20200312"), + ): + policy = client.generate_signed_post_policy( + BUCKET_NAME, + BLOB_NAME, + conditions=[], + expiration=datetime.datetime(2020, 3, 12), + virtual_hosted_style=True, + ) + self.assertEqual( + policy["url"], "https://{}.storage.googleapis.com".format(BUCKET_NAME) + ) + + def test_generate_signed_post_policy_bucket_bound_hostname(self): + import datetime + + BUCKET_NAME = "bucket-name" + BLOB_NAME = "object-name" + TIMESTAMP = "20200312T114716Z" + + credentials = _make_credentials() + credentials.sign_bytes = mock.Mock(return_value=b"Signature_bytes") + credentials.signer_email = "test@mail.com" + + client = self._make_one(project="PROJECT", credentials=credentials) + + with mock.patch( + "google.cloud.storage.client.get_v4_dtstamps", + return_value=(TIMESTAMP, "20200312"), + ): + policy = client.generate_signed_post_policy( + BUCKET_NAME, + BLOB_NAME, + conditions=[], + expiration=datetime.datetime(2020, 3, 12), + bucket_bound_hostname="https://bucket.bound_hostname", + ) + self.assertEqual( + policy["url"], "https://bucket.bound_hostname/{}".format(BUCKET_NAME) + ) + + def test_generate_signed_post_policy_bucket_bound_hostname_with_scheme(self): + import datetime + + BUCKET_NAME = "bucket-name" + BLOB_NAME = "object-name" + TIMESTAMP = "20200312T114716Z" + + credentials = _make_credentials() + credentials.sign_bytes = mock.Mock(return_value=b"Signature_bytes") + credentials.signer_email = "test@mail.com" + + client = self._make_one(project="PROJECT", credentials=credentials) + + with mock.patch( + "google.cloud.storage.client.get_v4_dtstamps", + return_value=(TIMESTAMP, "20200312"), + ): + policy = client.generate_signed_post_policy( + BUCKET_NAME, + BLOB_NAME, + conditions=[], + expiration=datetime.datetime(2020, 3, 12), + bucket_bound_hostname="bucket.bound_hostname", + scheme="http", + ) + self.assertEqual( + policy["url"], "http://bucket.bound_hostname/{}".format(BUCKET_NAME) + )