Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set default ZarrPlugin prefix to /zarr #188

Merged
merged 4 commits into from
May 5, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions tests/test_rest_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -288,42 +288,42 @@ def test_repr(airtemp_ds, airtemp_app_client):


def test_zmetadata(airtemp_ds, airtemp_app_client):
response = airtemp_app_client.get('/.zmetadata')
response = airtemp_app_client.get('/zarr/.zmetadata')
assert response.status_code == 200
assert json.dumps(response.json()) == json.dumps(
jsonify_zmetadata(airtemp_ds, create_zmetadata(airtemp_ds))
)


def test_bad_key(airtemp_app_client):
response = airtemp_app_client.get('/notakey')
response = airtemp_app_client.get('/zarr/notakey')
assert response.status_code == 404


def test_zgroup(airtemp_app_client):
response = airtemp_app_client.get('/.zgroup')
response = airtemp_app_client.get('/zarr/.zgroup')
assert response.status_code == 200


def test_zarray(airtemp_app_client):
response = airtemp_app_client.get('/air/.zarray')
response = airtemp_app_client.get('/zarr/air/.zarray')
assert response.status_code == 200


def test_zattrs(airtemp_app_client):
response = airtemp_app_client.get('/air/.zattrs')
response = airtemp_app_client.get('/zarr/air/.zattrs')
assert response.status_code == 200
response = airtemp_app_client.get('/.zattrs')
response = airtemp_app_client.get('/zarr/.zattrs')
assert response.status_code == 200


def test_get_chunk(airtemp_app_client):
response = airtemp_app_client.get('/air/0.0.0')
response = airtemp_app_client.get('/zarr/air/0.0.0')
assert response.status_code == 200


def test_array_group_raises_404(airtemp_app_client):
response = airtemp_app_client.get('/air/.zgroup')
response = airtemp_app_client.get('/zarr/air/.zgroup')
assert response.status_code == 404


Expand All @@ -333,12 +333,12 @@ def test_cache(airtemp_ds):

client = TestClient(rest.app)

response1 = client.get('/air/0.0.0')
response1 = client.get('/zarr/air/0.0.0')
assert response1.status_code == 200
assert '/air/0.0.0' in rest.cache

# test that we can retrieve
response2 = client.get('/air/0.0.0')
response2 = client.get('/zarr/air/0.0.0')
assert response2.status_code == 200
assert response1.content == response2.content

Expand Down Expand Up @@ -377,7 +377,7 @@ def test_ds_dict_keys(ds_dict, ds_dict_app_client):
assert response.status_code == 200
assert response.json() == list(ds_dict)

response = ds_dict_app_client.get('/datasets/not_in_dict')
response = ds_dict_app_client.get('/datasets/zarr/not_in_dict')
assert response.status_code == 404


Expand All @@ -386,7 +386,7 @@ def test_ds_dict_cache(ds_dict):

client = TestClient(rest.app)

response1 = client.get('/datasets/ds1/var/0')
response1 = client.get('/datasets/ds1/zarr/var/0')
assert response1.status_code == 200
assert 'ds1/var/0' in rest.cache

Expand Down
61 changes: 60 additions & 1 deletion tests/test_zarr_compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,19 @@ def test_zmetadata_identical(start, end, freq, nlats, nlons, var_const, calendar
@pytest.mark.parametrize(
'start, end, freq, nlats, nlons, var_const, calendar, use_cftime',
[
('2018-01-01', '2021-01-01', 'MS', 15, 30, True, 'standard', False),
pytest.param(
'2018-01-01',
'2021-01-01',
'MS',
15,
30,
True,
'standard',
False,
marks=pytest.mark.xfail(
reason="Xarray's ordering of metadata coords in a string can very"
),
),
],
)
def test_zmetadata_identical_coords(
Expand All @@ -65,6 +77,53 @@ def test_zmetadata_identical_coords(
mapper = TestMapper(SingleDatasetRest(ds).app)
actual = json.loads(mapper['.zmetadata'].decode())
expected = json.loads(zarr_dict['.zmetadata'].decode())
assert (
actual == expected
), "Zarr metadata did not match, likely because array coordinates aren't sorted by Xarray"


@pytest.mark.parametrize(
'start, end, freq, nlats, nlons, var_const, calendar, use_cftime',
[
(
'2018-01-01',
'2021-01-01',
'MS',
15,
30,
True,
'standard',
False,
),
],
)
def test_zmetadata_identical_coords_sorted(
start, end, freq, nlats, nlons, var_const, calendar, use_cftime
):
"""Test that zmetadata passes when coords are explicitly sorted"""
ds = create_dataset(
start=start,
end=end,
freq=freq,
nlats=nlats,
nlons=nlons,
var_const=var_const,
use_cftime=use_cftime,
calendar=calendar,
use_xy_dim=True,
)

ds = ds.chunk(ds.dims)
zarr_dict = {}
ds.to_zarr(zarr_dict, consolidated=True)
mapper = TestMapper(SingleDatasetRest(ds).app)
actual = json.loads(mapper['.zmetadata'].decode())
expected = json.loads(zarr_dict['.zmetadata'].decode())

for key in ['tmin/.zattrs', 'tmax/.zattrs']:
coords = expected['metadata'][key]['coordinates']
expected['metadata'][key]['coordinates'] = ' '.join(sorted(coords.split(' ')))

assert actual == expected


Expand Down
5 changes: 3 additions & 2 deletions tests/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,10 @@ class TestMapper(TestClient, BaseStore):
"""

def __getitem__(self, key):
response = self.get(key)
zarr_key = f'/zarr/{key}'
response = self.get(zarr_key)
if response.status_code != 200:
raise KeyError('{} not found. status_code = {}'.format(key, response.status_code))
raise KeyError('{} not found. status_code = {}'.format(zarr_key, response.status_code))
return response.content

def __delitem__(self, key):
Expand Down
2 changes: 1 addition & 1 deletion xpublish/plugins/included/zarr.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ class ZarrPlugin(Plugin):

name = 'zarr'

dataset_router_prefix: str = ''
dataset_router_prefix: str = '/zarr'
dataset_router_tags: Sequence[str] = ['zarr']

@hookimpl
Expand Down
2 changes: 1 addition & 1 deletion xpublish/utils/zarr.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ def _extract_dataarray_coords(da, zattrs):
nondim_coords = set(da.coords) - set(da.dims)

if len(nondim_coords) > 0 and da.name not in nondim_coords:
coords = ' '.join(list(nondim_coords))
coords = ' '.join(sorted(list(nondim_coords)))
zattrs['coordinates'] = encode_zarr_attr_value(coords)
return zattrs

Expand Down