Skip to content

Commit

Permalink
feat: add use_auth_w_custom_endpoint support (#941)
Browse files Browse the repository at this point in the history
* feat: add support for use_auth_w_custom_endpoint

* improve readability w bool _is_emulator_set

* update tests

Co-authored-by: Andrew Gorcester <gorcester@google.com>
  • Loading branch information
cojenco and andrewsg authored Dec 7, 2022
1 parent 64406ca commit 5291c08
Show file tree
Hide file tree
Showing 3 changed files with 135 additions and 44 deletions.
13 changes: 8 additions & 5 deletions google/cloud/storage/_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,17 +33,20 @@
STORAGE_EMULATOR_ENV_VAR = "STORAGE_EMULATOR_HOST"
"""Environment variable defining host for Storage emulator."""

_API_ENDPOINT_OVERRIDE_ENV_VAR = "API_ENDPOINT_OVERRIDE"
"""This is an experimental configuration variable. Use api_endpoint instead."""

_API_VERSION_OVERRIDE_ENV_VAR = "API_VERSION_OVERRIDE"
"""This is an experimental configuration variable used for internal testing."""

_DEFAULT_STORAGE_HOST = os.getenv(
"API_ENDPOINT_OVERRIDE", "https://storage.googleapis.com"
_API_ENDPOINT_OVERRIDE_ENV_VAR, "https://storage.googleapis.com"
)
"""Default storage host for JSON API."""

_API_VERSION = os.getenv("API_VERSION_OVERRIDE", "v1")
_API_VERSION = os.getenv(_API_VERSION_OVERRIDE_ENV_VAR, "v1")
"""API version of the default storage host"""

_BASE_STORAGE_URI = "storage.googleapis.com"
"""Base request endpoint URI for JSON API."""

# etag match parameters in snake case and equivalent header
_ETAG_MATCH_PARAMETERS = (
("if_etag_match", "If-Match"),
Expand Down
48 changes: 28 additions & 20 deletions google/cloud/storage/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
from google.cloud.storage._helpers import _get_default_headers
from google.cloud.storage._helpers import _get_environ_project
from google.cloud.storage._helpers import _get_storage_host
from google.cloud.storage._helpers import _BASE_STORAGE_URI
from google.cloud.storage._helpers import _DEFAULT_STORAGE_HOST
from google.cloud.storage._helpers import _bucket_bound_hostname_url
from google.cloud.storage._helpers import _add_etag_match_headers
Expand Down Expand Up @@ -96,6 +95,12 @@ class Client(ClientWithProject):
:type client_options: :class:`~google.api_core.client_options.ClientOptions` or :class:`dict`
:param client_options: (Optional) Client options used to set user options on the client.
API Endpoint should be set through client_options.
:type use_auth_w_custom_endpoint: bool
:param use_auth_w_custom_endpoint:
(Optional) Whether authentication is required under custom endpoints.
If false, uses AnonymousCredentials and bypasses authentication.
Defaults to True. Note this is only used when a custom endpoint is set in conjunction.
"""

SCOPE = (
Expand All @@ -112,6 +117,7 @@ def __init__(
_http=None,
client_info=None,
client_options=None,
use_auth_w_custom_endpoint=True,
):
self._base_connection = None

Expand All @@ -127,13 +133,12 @@ def __init__(
kw_args = {"client_info": client_info}

# `api_endpoint` should be only set by the user via `client_options`,
# or if the _get_storage_host() returns a non-default value.
# or if the _get_storage_host() returns a non-default value (_is_emulator_set).
# `api_endpoint` plays an important role for mTLS, if it is not set,
# then mTLS logic will be applied to decide which endpoint will be used.
storage_host = _get_storage_host()
kw_args["api_endpoint"] = (
storage_host if storage_host != _DEFAULT_STORAGE_HOST else None
)
_is_emulator_set = storage_host != _DEFAULT_STORAGE_HOST
kw_args["api_endpoint"] = storage_host if _is_emulator_set else None

if client_options:
if type(client_options) == dict:
Expand All @@ -144,19 +149,20 @@ def __init__(
api_endpoint = client_options.api_endpoint
kw_args["api_endpoint"] = api_endpoint

# Use anonymous credentials and no project when
# STORAGE_EMULATOR_HOST or a non-default api_endpoint is set.
if (
kw_args["api_endpoint"] is not None
and _BASE_STORAGE_URI not in kw_args["api_endpoint"]
):
if credentials is None:
credentials = AnonymousCredentials()
if project is None:
project = _get_environ_project()
if project is None:
no_project = True
project = "<none>"
# If a custom endpoint is set, the client checks for credentials
# or finds the default credentials based on the current environment.
# Authentication may be bypassed under certain conditions:
# (1) STORAGE_EMULATOR_HOST is set (for backwards compatibility), OR
# (2) use_auth_w_custom_endpoint is set to False.
if kw_args["api_endpoint"] is not None:
if _is_emulator_set or not use_auth_w_custom_endpoint:
if credentials is None:
credentials = AnonymousCredentials()
if project is None:
project = _get_environ_project()
if project is None:
no_project = True
project = "<none>"

super(Client, self).__init__(
project=project,
Expand Down Expand Up @@ -897,7 +903,8 @@ def create_bucket(
project = self.project

# Use no project if STORAGE_EMULATOR_HOST is set
if _BASE_STORAGE_URI not in _get_storage_host():
_is_emulator_set = _get_storage_host() != _DEFAULT_STORAGE_HOST
if _is_emulator_set:
if project is None:
project = _get_environ_project()
if project is None:
Expand Down Expand Up @@ -1327,7 +1334,8 @@ def list_buckets(
project = self.project

# Use no project if STORAGE_EMULATOR_HOST is set
if _BASE_STORAGE_URI not in _get_storage_host():
_is_emulator_set = _get_storage_host() != _DEFAULT_STORAGE_HOST
if _is_emulator_set:
if project is None:
project = _get_environ_project()
if project is None:
Expand Down
118 changes: 99 additions & 19 deletions tests/unit/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,10 @@
from google.auth.credentials import AnonymousCredentials
from google.oauth2.service_account import Credentials

from google.cloud.storage import _helpers
from google.cloud.storage._helpers import STORAGE_EMULATOR_ENV_VAR
from google.cloud.storage._helpers import _get_default_headers
from google.cloud.storage import _helpers
from google.cloud.storage._http import Connection
from google.cloud.storage.retry import DEFAULT_RETRY
from google.cloud.storage.retry import DEFAULT_RETRY_IF_GENERATION_SPECIFIED
from tests.unit.test__helpers import GCCL_INVOCATION_TEST_CONST
Expand Down Expand Up @@ -119,7 +120,6 @@ def _make_one(self, *args, **kw):

def test_ctor_connection_type(self):
from google.cloud._http import ClientInfo
from google.cloud.storage._http import Connection

PROJECT = "PROJECT"
credentials = _make_credentials()
Expand Down Expand Up @@ -179,8 +179,6 @@ def test_ctor_w_client_options_object(self):
)

def test_ctor_wo_project(self):
from google.cloud.storage._http import Connection

PROJECT = "PROJECT"
credentials = _make_credentials(project=PROJECT)

Expand All @@ -193,8 +191,6 @@ def test_ctor_wo_project(self):
self.assertEqual(list(client._batch_stack), [])

def test_ctor_w_project_explicit_none(self):
from google.cloud.storage._http import Connection

credentials = _make_credentials()

client = self._make_one(project=None, credentials=credentials)
Expand All @@ -207,7 +203,6 @@ def test_ctor_w_project_explicit_none(self):

def test_ctor_w_client_info(self):
from google.cloud._http import ClientInfo
from google.cloud.storage._http import Connection

credentials = _make_credentials()
client_info = ClientInfo()
Expand Down Expand Up @@ -239,8 +234,40 @@ def test_ctor_mtls(self):
self.assertEqual(client._connection.ALLOW_AUTO_SWITCH_TO_MTLS_URL, False)
self.assertEqual(client._connection.API_BASE_URL, "http://foo")

def test_ctor_w_custom_endpoint_use_auth(self):
custom_endpoint = "storage-example.p.googleapis.com"
client = self._make_one(client_options={"api_endpoint": custom_endpoint})
self.assertEqual(client._connection.API_BASE_URL, custom_endpoint)
self.assertIsNotNone(client.project)
self.assertIsInstance(client._connection, Connection)
self.assertIsNotNone(client._connection.credentials)
self.assertNotIsInstance(client._connection.credentials, AnonymousCredentials)

def test_ctor_w_custom_endpoint_bypass_auth(self):
custom_endpoint = "storage-example.p.googleapis.com"
client = self._make_one(
client_options={"api_endpoint": custom_endpoint},
use_auth_w_custom_endpoint=False,
)
self.assertEqual(client._connection.API_BASE_URL, custom_endpoint)
self.assertEqual(client.project, None)
self.assertIsInstance(client._connection, Connection)
self.assertIsInstance(client._connection.credentials, AnonymousCredentials)

def test_ctor_w_custom_endpoint_w_credentials(self):
PROJECT = "PROJECT"
custom_endpoint = "storage-example.p.googleapis.com"
credentials = _make_credentials(project=PROJECT)
client = self._make_one(
credentials=credentials, client_options={"api_endpoint": custom_endpoint}
)
self.assertEqual(client._connection.API_BASE_URL, custom_endpoint)
self.assertEqual(client.project, PROJECT)
self.assertIsInstance(client._connection, Connection)
self.assertIs(client._connection.credentials, credentials)

def test_ctor_w_emulator_wo_project(self):
# avoids authentication if STORAGE_EMULATOR_ENV_VAR is set
# bypasses authentication if STORAGE_EMULATOR_ENV_VAR is set
host = "http://localhost:8080"
environ = {STORAGE_EMULATOR_ENV_VAR: host}
with mock.patch("os.environ", environ):
Expand All @@ -250,16 +277,8 @@ def test_ctor_w_emulator_wo_project(self):
self.assertEqual(client._connection.API_BASE_URL, host)
self.assertIsInstance(client._connection.credentials, AnonymousCredentials)

# avoids authentication if storage emulator is set through api_endpoint
client = self._make_one(
client_options={"api_endpoint": "http://localhost:8080"}
)
self.assertIsNone(client.project)
self.assertEqual(client._connection.API_BASE_URL, host)
self.assertIsInstance(client._connection.credentials, AnonymousCredentials)

def test_ctor_w_emulator_w_environ_project(self):
# avoids authentication and infers the project from the environment
# bypasses authentication and infers the project from the environment
host = "http://localhost:8080"
environ_project = "environ-project"
environ = {
Expand Down Expand Up @@ -289,9 +308,17 @@ def test_ctor_w_emulator_w_project_arg(self):
self.assertEqual(client._connection.API_BASE_URL, host)
self.assertIsInstance(client._connection.credentials, AnonymousCredentials)

def test_create_anonymous_client(self):
from google.cloud.storage._http import Connection
def test_ctor_w_emulator_w_credentials(self):
host = "http://localhost:8080"
environ = {STORAGE_EMULATOR_ENV_VAR: host}
credentials = _make_credentials()
with mock.patch("os.environ", environ):
client = self._make_one(credentials=credentials)

self.assertEqual(client._connection.API_BASE_URL, host)
self.assertIs(client._connection.credentials, credentials)

def test_create_anonymous_client(self):
klass = self._get_target_class()
client = klass.create_anonymous_client()

Expand Down Expand Up @@ -1269,6 +1296,28 @@ def test_create_bucket_w_environ_project_w_emulator(self):
_target_object=bucket,
)

def test_create_bucket_w_custom_endpoint(self):
custom_endpoint = "storage-example.p.googleapis.com"
client = self._make_one(client_options={"api_endpoint": custom_endpoint})
bucket_name = "bucket-name"
api_response = {"name": bucket_name}
client._post_resource = mock.Mock()
client._post_resource.return_value = api_response

bucket = client.create_bucket(bucket_name)

expected_path = "/b"
expected_data = api_response
expected_query_params = {"project": client.project}
client._post_resource.assert_called_once_with(
expected_path,
expected_data,
query_params=expected_query_params,
timeout=self._get_default_timeout(),
retry=DEFAULT_RETRY,
_target_object=bucket,
)

def test_create_bucket_w_conflict_w_user_project(self):
from google.cloud.exceptions import Conflict

Expand Down Expand Up @@ -2055,6 +2104,37 @@ def test_list_buckets_w_environ_project_w_emulator(self):
retry=DEFAULT_RETRY,
)

def test_list_buckets_w_custom_endpoint(self):
from google.cloud.storage.client import _item_to_bucket

custom_endpoint = "storage-example.p.googleapis.com"
client = self._make_one(client_options={"api_endpoint": custom_endpoint})
client._list_resource = mock.Mock(spec=[])

iterator = client.list_buckets()

self.assertIs(iterator, client._list_resource.return_value)

expected_path = "/b"
expected_item_to_value = _item_to_bucket
expected_page_token = None
expected_max_results = None
expected_page_size = None
expected_extra_params = {
"project": client.project,
"projection": "noAcl",
}
client._list_resource.assert_called_once_with(
expected_path,
expected_item_to_value,
page_token=expected_page_token,
max_results=expected_max_results,
extra_params=expected_extra_params,
page_size=expected_page_size,
timeout=self._get_default_timeout(),
retry=DEFAULT_RETRY,
)

def test_list_buckets_w_defaults(self):
from google.cloud.storage.client import _item_to_bucket

Expand Down

0 comments on commit 5291c08

Please sign in to comment.