Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Storage: add 'Client.list_blobs(bucket_or_name)'. #8375

Merged
merged 10 commits into from
Jun 28, 2019
Merged

Storage: add 'Client.list_blobs(bucket_or_name)'. #8375

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ad54362
Union create-or-use bucket snippets into one helper
Jun 14, 2019
1835337
Deleted test_list_blobs - looks like it's a full copy of test_list_bl…
Jun 14, 2019
e038d19
Add new method client.list_blobs that's the same as bucket.list_blobs()
Jun 17, 2019
107a02b
Merge branch 'master' into storage_client_redesign_7762
Jun 17, 2019
e120a23
Redact comment and delete unittest runner
Jun 17, 2019
8175788
Add deprecation for bucket.list_blobs
Jun 17, 2019
7e20131
Changed warn to comment
Jun 17, 2019
04fc185
Merge branch 'master' into storage_client_redesign_7762
Jun 24, 2019
f083156
Bring method docs to Google-style, use mock connection instead of class
Jun 24, 2019
42b1a40
Nit on docing
Jun 28, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions storage/google/cloud/storage/bucket.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -721,6 +721,9 @@ def list_blobs(
):
"""Return an iterator used to find blobs in the bucket.

.. note::
Direct use of this method is deprecated. Use Client.list_blobs instead.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit. Add backticks for sphinx to render as code font.

Use ``Client.list_blobs`` instead.


If :attr:`user_project` is set, bills the API request to that project.

:type max_results: int
Expand Down Expand Up @@ -770,11 +773,6 @@ def list_blobs(
:returns: Iterator of all :class:`~google.cloud.storage.blob.Blob`
in this bucket matching the arguments.
"""
warnings.warn(
"Bucket.list_blobs method is deprecated. Use Client.list_blobs instead.",
PendingDeprecationWarning,
stacklevel=2,
)
extra_params = {"projection": projection}

if prefix is not None:
Expand Down
98 changes: 48 additions & 50 deletions storage/google/cloud/storage/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ def _pop_batch(self):
"""
return self._batch_stack.pop()

def _use_or_init_bucket(self, bucket_or_name):
def _bucket_arg_to_bucket(self, bucket_or_name):
"""Helper to return given bucket or create new by name.

Args:
Expand Down Expand Up @@ -272,7 +272,7 @@ def get_bucket(self, bucket_or_name):
>>> bucket = client.get_bucket(bucket) # API request.

"""
bucket = self._use_or_init_bucket(bucket_or_name)
bucket = self._bucket_arg_to_bucket(bucket_or_name)

bucket.reload(client=self)
return bucket
Expand Down Expand Up @@ -346,7 +346,7 @@ def create_bucket(self, bucket_or_name, requester_pays=None, project=None):
>>> bucket = client.create_bucket(bucket) # API request.

"""
bucket = self._use_or_init_bucket(bucket_or_name)
bucket = self._bucket_arg_to_bucket(bucket_or_name)

if requester_pays is not None:
bucket.requester_pays = requester_pays
Expand Down Expand Up @@ -419,56 +419,54 @@ def list_blobs(

If :attr:`user_project` is set, bills the API request to that project.

bucket_or_name (Union[ \
:class:`~google.cloud.storage.bucket.Bucket`, \
str, \
]):
The bucket resource to pass or name to create.

:type max_results: int
:param max_results:
(Optional) The maximum number of blobs in each page of results
from this request. Non-positive values are ignored. Defaults to
a sensible value set by the API.

:type page_token: str
:param page_token:
(Optional) If present, return the next batch of blobs, using the
value, which must correspond to the ``nextPageToken`` value
returned in the previous response. Deprecated: use the ``pages``
property of the returned iterator instead of manually passing the
token.

:type prefix: str
:param prefix: (Optional) prefix used to filter blobs.

:type delimiter: str
:param delimiter: (Optional) Delimiter, used with ``prefix`` to
emulate hierarchy.

:type versions: bool
:param versions: (Optional) Whether object versions should be returned
as separate blobs.

:type projection: str
:param projection: (Optional) If used, must be 'full' or 'noAcl'.
Defaults to ``'noAcl'``. Specifies the set of
properties to return.
Args:
bucket_or_name (Union[ \
:class:`~google.cloud.storage.bucket.Bucket`, \
str, \
]):
The bucket resource to pass or name to create.

:type fields: str
:param fields:
(Optional) Selector specifying which fields to include
in a partial response. Must be a list of fields. For
example to get a partial response with just the next
page token and the name and language of each blob returned:
``'items(name,contentLanguage),nextPageToken'``.
See: https://cloud.google.com/storage/docs/json_api/v1/parameters#fields
max_results (int):
(Optional) The maximum number of blobs in each page of results
from this request. Non-positive values are ignored. Defaults to
a sensible value set by the API.

page_token (str):
(Optional) If present, return the next batch of blobs, using the
value, which must correspond to the ``nextPageToken`` value
returned in the previous response. Deprecated: use the ``pages``
property of the returned iterator instead of manually passing the
token.

prefix (str):
(Optional) prefix used to filter blobs.

delimiter (str):
(Optional) Delimiter, used with ``prefix`` to
emulate hierarchy.

versions (bool):
(Optional) Whether object versions should be returned
as separate blobs.

projection (str):
(Optional) If used, must be 'full' or 'noAcl'.
Defaults to ``'noAcl'``. Specifies the set of
properties to return.

fields (str):
(Optional) Selector specifying which fields to include
in a partial response. Must be a list of fields. For
example to get a partial response with just the next
page token and the name and language of each blob returned:
``'items(name,contentLanguage),nextPageToken'``.
See: https://cloud.google.com/storage/docs/json_api/v1/parameters#fields

:rtype: :class:`~google.api_core.page_iterator.Iterator`
:returns: Iterator of all :class:`~google.cloud.storage.blob.Blob`
in this bucket matching the arguments.
Returns:
Iterator of all :class:`~google.cloud.storage.blob.Blob`
in this bucket matching the arguments.
"""
bucket = self._use_or_init_bucket(bucket_or_name)
bucket = self._bucket_arg_to_bucket(bucket_or_name)
return bucket.list_blobs(
max_results=max_results,
page_token=page_token,
Expand Down
68 changes: 22 additions & 46 deletions storage/tests/unit/test_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,16 @@ def _make_credentials():
return mock.Mock(spec=google.auth.credentials.Credentials)


def _make_connection(*responses):
import google.cloud.storage._http
from google.cloud.exceptions import NotFound

mock_conn = mock.create_autospec(google.cloud.storage._http.Connection)
mock_conn.user_agent = "testing 1.2.3"
mock_conn.api_request.side_effect = list(responses) + [NotFound("miss")]
return mock_conn


def _make_response(status=http_client.OK, content=b"", headers={}):
response = requests.Response()
response.status_code = status
Expand Down Expand Up @@ -645,7 +655,7 @@ def test_list_blobs(self):

credentials = _make_credentials()
client = self._make_one(project="PROJECT", credentials=credentials)
connection = _Connection({"items": []})
connection = _make_connection({"items": []})

with mock.patch(
'google.cloud.storage.client.Client._connection',
Expand All @@ -658,10 +668,11 @@ def test_list_blobs(self):
blobs = list(iterator)

self.assertEqual(blobs, [])
kw, = connection._requested
self.assertEqual(kw["method"], "GET")
self.assertEqual(kw["path"], "/b/%s/o" % BUCKET_NAME)
self.assertEqual(kw["query_params"], {"projection": "noAcl"})
connection.api_request.assert_called_once_with(
method="GET",
path="/b/%s/o" % BUCKET_NAME,
query_params={"projection": "noAcl"}
)

def test_list_blobs_w_all_arguments_and_user_project(self):
from google.cloud.storage.bucket import Bucket
Expand All @@ -687,7 +698,7 @@ def test_list_blobs_w_all_arguments_and_user_project(self):

credentials = _make_credentials()
client = self._make_one(project=USER_PROJECT, credentials=credentials)
connection = _Connection({"items": []})
connection = _make_connection({"items": []})

with mock.patch(
'google.cloud.storage.client.Client._connection',
Expand All @@ -709,10 +720,11 @@ def test_list_blobs_w_all_arguments_and_user_project(self):
blobs = list(iterator)

self.assertEqual(blobs, [])
kw, = connection._requested
self.assertEqual(kw["method"], "GET")
self.assertEqual(kw["path"], "/b/%s/o" % BUCKET_NAME)
self.assertEqual(kw["query_params"], EXPECTED)
connection.api_request.assert_called_once_with(
method="GET",
path="/b/%s/o" % BUCKET_NAME,
query_params=EXPECTED
)

def test_list_buckets_wo_project(self):
CREDENTIALS = _make_credentials()
Expand Down Expand Up @@ -896,39 +908,3 @@ def dummy_response():
self.assertEqual(page.remaining, 0)
self.assertIsInstance(bucket, Bucket)
self.assertEqual(bucket.name, blob_name)


class _Connection(object):
_delete_bucket = False

def __init__(self, *responses):
self._responses = responses
self._requested = []
self._deleted_buckets = []
self.credentials = None

@staticmethod
def _is_bucket_path(path):
# Now just ensure the path only has /b/ and one more segment.
return path.startswith("/b/") and path.count("/") == 2

def api_request(self, **kw):
from google.cloud.exceptions import NotFound

self._requested.append(kw)

method = kw.get("method")
path = kw.get("path", "")
if method == "DELETE" and self._is_bucket_path(path):
self._deleted_buckets.append(kw)
if self._delete_bucket:
return
else:
raise NotFound("miss")

try:
response, self._responses = self._responses[0], self._responses[1:]
except IndexError:
raise NotFound("miss")
else:
return response