Skip to content

Commit

Permalink
feat: add timeout parameter to public methods (#44)
Browse files Browse the repository at this point in the history
* Add a default request timeout constant

* Add timeout to ACL public methods

* Add timeout to Blob methods

* Add default timeout to Batch._do_request()

* Add timeout to Bucket methods

* Add timeout to Client methods

* Add timeout to HMACKeyMetadata methods

* Add timeout to BucketNotification methods

* Add timeout to _PropertyMixin helpers
  • Loading branch information
plamut authored Feb 11, 2020
1 parent 64abf24 commit 63abf07
Show file tree
Hide file tree
Showing 18 changed files with 869 additions and 202 deletions.
30 changes: 27 additions & 3 deletions google/cloud/storage/_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@
from hashlib import md5
import os

from google.cloud.storage.constants import _DEFAULT_TIMEOUT


STORAGE_EMULATOR_ENV_VAR = "STORAGE_EMULATOR_HOST"
"""Environment variable defining host for Storage emulator."""

Expand Down Expand Up @@ -117,7 +120,7 @@ def _query_params(self):
params["userProject"] = self.user_project
return params

def reload(self, client=None):
def reload(self, client=None, timeout=_DEFAULT_TIMEOUT):
"""Reload properties from Cloud Storage.
If :attr:`user_project` is set, bills the API request to that project.
Expand All @@ -126,6 +129,12 @@ def reload(self, client=None):
``NoneType``
:param client: the client to use. If not passed, falls back to the
``client`` stored on the current object.
:type timeout: float or tuple
:param timeout: (optional) The amount of time, in seconds, to wait
for the server response.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
"""
client = self._require_client(client)
query_params = self._query_params
Expand All @@ -138,6 +147,7 @@ def reload(self, client=None):
query_params=query_params,
headers=self._encryption_headers(),
_target_object=self,
timeout=timeout,
)
self._set_properties(api_response)

Expand Down Expand Up @@ -169,7 +179,7 @@ def _set_properties(self, value):
# If the values are reset, the changes must as well.
self._changes = set()

def patch(self, client=None):
def patch(self, client=None, timeout=_DEFAULT_TIMEOUT):
"""Sends all changed properties in a PATCH request.
Updates the ``_properties`` with the response from the backend.
Expand All @@ -180,6 +190,12 @@ def patch(self, client=None):
``NoneType``
:param client: the client to use. If not passed, falls back to the
``client`` stored on the current object.
:type timeout: float or tuple
:param timeout: (optional) The amount of time, in seconds, to wait
for the server response.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
"""
client = self._require_client(client)
query_params = self._query_params
Expand All @@ -195,10 +211,11 @@ def patch(self, client=None):
data=update_properties,
query_params=query_params,
_target_object=self,
timeout=timeout,
)
self._set_properties(api_response)

def update(self, client=None):
def update(self, client=None, timeout=_DEFAULT_TIMEOUT):
"""Sends all properties in a PUT request.
Updates the ``_properties`` with the response from the backend.
Expand All @@ -209,6 +226,12 @@ def update(self, client=None):
``NoneType``
:param client: the client to use. If not passed, falls back to the
``client`` stored on the current object.
:type timeout: float or tuple
:param timeout: (optional) The amount of time, in seconds, to wait
for the server response.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
"""
client = self._require_client(client)
query_params = self._query_params
Expand All @@ -219,6 +242,7 @@ def update(self, client=None):
data=self._properties,
query_params=query_params,
_target_object=self,
timeout=timeout,
)
self._set_properties(api_response)

Expand Down
65 changes: 53 additions & 12 deletions google/cloud/storage/acl.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@
when sending metadata for ACLs to the API.
"""

from google.cloud.storage.constants import _DEFAULT_TIMEOUT


class _ACLEntity(object):
"""Class representing a set of roles for an entity.
Expand Down Expand Up @@ -206,10 +208,18 @@ class ACL(object):
def __init__(self):
self.entities = {}

def _ensure_loaded(self):
"""Load if not already loaded."""
def _ensure_loaded(self, timeout=_DEFAULT_TIMEOUT):
"""Load if not already loaded.
:type timeout: float or tuple
:param timeout: (optional) The amount of time, in seconds, to wait
for the server response.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
"""
if not self.loaded:
self.reload()
self.reload(timeout=timeout)

@classmethod
def validate_predefined(cls, predefined):
Expand Down Expand Up @@ -415,7 +425,7 @@ def _require_client(self, client):
client = self.client
return client

def reload(self, client=None):
def reload(self, client=None, timeout=_DEFAULT_TIMEOUT):
"""Reload the ACL data from Cloud Storage.
If :attr:`user_project` is set, bills the API request to that project.
Expand All @@ -424,6 +434,12 @@ def reload(self, client=None):
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
to the ``client`` stored on the ACL's parent.
:type timeout: float or tuple
:param timeout: (optional) The amount of time, in seconds, to wait
for the server response.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
"""
path = self.reload_path
client = self._require_client(client)
Expand All @@ -435,13 +451,13 @@ def reload(self, client=None):
self.entities.clear()

found = client._connection.api_request(
method="GET", path=path, query_params=query_params
method="GET", path=path, query_params=query_params, timeout=timeout
)
self.loaded = True
for entry in found.get("items", ()):
self.add_entity(self.entity_from_dict(entry))

def _save(self, acl, predefined, client):
def _save(self, acl, predefined, client, timeout=_DEFAULT_TIMEOUT):
"""Helper for :meth:`save` and :meth:`save_predefined`.
:type acl: :class:`google.cloud.storage.acl.ACL`, or a compatible list.
Expand All @@ -457,6 +473,12 @@ def _save(self, acl, predefined, client):
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
to the ``client`` stored on the ACL's parent.
:type timeout: float or tuple
:param timeout: (optional) The amount of time, in seconds, to wait
for the server response.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
"""
query_params = {"projection": "full"}
if predefined is not None:
Expand All @@ -474,13 +496,14 @@ def _save(self, acl, predefined, client):
path=path,
data={self._URL_PATH_ELEM: list(acl)},
query_params=query_params,
timeout=timeout,
)
self.entities.clear()
for entry in result.get(self._URL_PATH_ELEM, ()):
self.add_entity(self.entity_from_dict(entry))
self.loaded = True

def save(self, acl=None, client=None):
def save(self, acl=None, client=None, timeout=_DEFAULT_TIMEOUT):
"""Save this ACL for the current bucket.
If :attr:`user_project` is set, bills the API request to that project.
Expand All @@ -493,6 +516,12 @@ def save(self, acl=None, client=None):
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
to the ``client`` stored on the ACL's parent.
:type timeout: float or tuple
:param timeout: (optional) The amount of time, in seconds, to wait
for the server response.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
"""
if acl is None:
acl = self
Expand All @@ -501,9 +530,9 @@ def save(self, acl=None, client=None):
save_to_backend = True

if save_to_backend:
self._save(acl, None, client)
self._save(acl, None, client, timeout=timeout)

def save_predefined(self, predefined, client=None):
def save_predefined(self, predefined, client=None, timeout=_DEFAULT_TIMEOUT):
"""Save this ACL for the current bucket using a predefined ACL.
If :attr:`user_project` is set, bills the API request to that project.
Expand All @@ -519,11 +548,17 @@ def save_predefined(self, predefined, client=None):
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
to the ``client`` stored on the ACL's parent.
:type timeout: float or tuple
:param timeout: (optional) The amount of time, in seconds, to wait
for the server response.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
"""
predefined = self.validate_predefined(predefined)
self._save(None, predefined, client)
self._save(None, predefined, client, timeout=timeout)

def clear(self, client=None):
def clear(self, client=None, timeout=_DEFAULT_TIMEOUT):
"""Remove all ACL entries.
If :attr:`user_project` is set, bills the API request to that project.
Expand All @@ -537,8 +572,14 @@ def clear(self, client=None):
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
to the ``client`` stored on the ACL's parent.
:type timeout: float or tuple
:param timeout: (optional) The amount of time, in seconds, to wait
for the server response.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
"""
self.save([], client=client)
self.save([], client=client, timeout=timeout)


class BucketACL(ACL):
Expand Down
12 changes: 8 additions & 4 deletions google/cloud/storage/batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
from google.cloud import _helpers
from google.cloud import exceptions
from google.cloud.storage._http import Connection
from google.cloud.storage.constants import _DEFAULT_TIMEOUT


class MIMEApplicationHTTP(MIMEApplication):
Expand Down Expand Up @@ -150,7 +151,9 @@ def __init__(self, client):
self._requests = []
self._target_objects = []

def _do_request(self, method, url, headers, data, target_object, timeout=None):
def _do_request(
self, method, url, headers, data, target_object, timeout=_DEFAULT_TIMEOUT
):
"""Override Connection: defer actual HTTP request.
Only allow up to ``_MAX_BATCH_SIZE`` requests to be deferred.
Expand All @@ -175,7 +178,8 @@ def _do_request(self, method, url, headers, data, target_object, timeout=None):
:type timeout: float or tuple
:param timeout: (optional) The amount of time, in seconds, to wait
for the server response. By default, the method waits indefinitely.
for the server response.
Can also be passed as a tuple (connect_timeout, read_timeout).
See :meth:`requests.Session.request` documentation for details.
Expand Down Expand Up @@ -206,8 +210,8 @@ def _prepare_batch_request(self):

multi = MIMEMultipart()

# Use timeout of last request, default to None (indefinite)
timeout = None
# Use timeout of last request, default to _DEFAULT_TIMEOUT
timeout = _DEFAULT_TIMEOUT
for method, uri, headers, body, _timeout in self._requests:
subrequest = MIMEApplicationHTTP(method, uri, headers, body)
multi.attach(subrequest)
Expand Down
Loading

0 comments on commit 63abf07

Please sign in to comment.