Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

generalize dims as X/Y when lat/lon or rlat/rlon #221

Merged
merged 21 commits into from
Jul 27, 2023
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
bf7795c
generalize dims as X/Y when lat/lon or rlat/rlon
vindelico Jul 5, 2023
7611f83
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 6, 2023
ee8e507
updated AUTHORS.rst and HISTORY.rst
vindelico Jul 6, 2023
a739554
put code changes and author/history changes together
vindelico Jul 6, 2023
b2509b7
update .zenodo.json
vindelico Jul 6, 2023
1ed3f32
Update xscen/io.py
vindelico Jul 10, 2023
c63b005
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 10, 2023
82ab63a
Update xscen/io.py
vindelico Jul 10, 2023
4fe31a4
Update HISTORY.rst
vindelico Jul 10, 2023
aea398e
Update HISTORY.rst
vindelico Jul 10, 2023
9b28425
Merge remote-tracking branch 'origin/main' into generalize_rlatrlon_l…
vindelico Jul 10, 2023
fafe3ec
documentation and tests for generalize_rlatrlon_latlon
vindelico Jul 24, 2023
412309f
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 24, 2023
ba67c10
Update docs/notebooks/2_getting_started.ipynb
vindelico Jul 25, 2023
ee324d2
Update xscen/io.py
vindelico Jul 25, 2023
ab3c08f
repairs following github discussion
vindelico Jul 25, 2023
f9be133
repairs following github discussion 2
vindelico Jul 25, 2023
112abc8
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jul 25, 2023
633469d
Update HISTORY.rst
vindelico Jul 25, 2023
bf9d906
reformatted tests
RondeauG Jul 25, 2023
7b93897
Merge remote-tracking branch 'origin/main' into generalize_rlatrlon_l…
RondeauG Jul 25, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .zenodo.json
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,11 @@
"name": "Caron, Louis-Philippe",
"affiliation": "Ouranos",
"orcid": "0000-0001-5221-0147"
},
{
"name": "Braun, Marco",
"affiliation": "Ouranos",
"orcid": "0000-0001-5061-3217"
}
],
"keywords": [
Expand Down
1 change: 1 addition & 0 deletions AUTHORS.rst
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,4 @@ Contributors
* Louis-Philippe Caron <caron.louis-philippe@ouranos.ca>
* Sarah Gammon <gammon.sarah@ouranos.ca> `@SarahG-579462 <https://github.com/SarahG-579462>`_
* Yannick Rousseau
* Marco Braun <Braun.Marco@ouranos.ca> `@vindelico <https://github.com/vindelico>`_
1 change: 1 addition & 0 deletions HISTORY.rst
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ New features and enhancements
* New function `get_warming_level` to search within the IPCC CMIP global temperatures CSV without requiring data. (:issue:`208`, :pull:`210`).
* File re-structuration from catalogs with ``xscen.catutils.build_path``. (:pull:`205`).
* New scripting functions `save_and_update` and `move_and_delete`. (:pull:`214`).
* Spatial dimensions can be generalized as X/Y when rechunking and will be mapped to rlon/rlat or lon/lat accordingly. (:pull: `221`).
vindelico marked this conversation as resolved.
Show resolved Hide resolved

Breaking changes
^^^^^^^^^^^^^^^^
Expand Down
58 changes: 26 additions & 32 deletions xscen/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -304,22 +304,7 @@ def save_to_netcdf(
xarray.Dataset.to_netcdf
"""
if rechunk:
for rechunk_var in ds.data_vars:
# Support for chunks varying per variable
if rechunk_var in rechunk:
rechunk_dims = rechunk[rechunk_var]
else:
rechunk_dims = rechunk

ds[rechunk_var] = ds[rechunk_var].chunk(
{
d: chnks
for d, chnks in rechunk_dims.items()
if d in ds[rechunk_var].dims
}
)
ds[rechunk_var].encoding.pop("chunksizes", None)
ds[rechunk_var].encoding.pop("chunks", None)
ds = _rechunk_for_saving(ds, rechunk)

path = Path(filename)
path.parent.mkdir(parents=True, exist_ok=True)
Expand Down Expand Up @@ -404,22 +389,7 @@ def save_to_zarr(
ds[v].encoding.clear()

if rechunk:
for rechunk_var in ds.data_vars:
# Support for chunks varying per variable
if rechunk_var in rechunk:
rechunk_dims = rechunk[rechunk_var]
else:
rechunk_dims = rechunk

ds[rechunk_var] = ds[rechunk_var].chunk(
{
d: chnks
for d, chnks in rechunk_dims.items()
if d in ds[rechunk_var].dims
}
)
ds[rechunk_var].encoding.pop("chunksizes", None)
ds[rechunk_var].encoding.pop("chunks", None)
ds = _rechunk_for_saving(ds, rechunk)

path = Path(filename)
path.parent.mkdir(parents=True, exist_ok=True)
Expand Down Expand Up @@ -518,6 +488,30 @@ def coerce_attrs(attrs):
raise


def _rechunk_for_saving(ds, rechunk):
vindelico marked this conversation as resolved.
Show resolved Hide resolved
"""Rechunk before saving to .zarr or .nc, generalized as Y/X for different axes lat/lon, rlat/rlon."""
for rechunk_var in ds.data_vars:
# Support for chunks varying per variable
if rechunk_var in rechunk:
rechunk_dims = rechunk[rechunk_var].copy()
else:
rechunk_dims = rechunk.copy()

# get actual axes labels
if "X" in rechunk_dims and "X" not in ds.dims:
rechunk_dims[ds.cf.axes["X"][0]] = rechunk_dims.pop("X")
if "Y" in rechunk_dims and "Y" not in ds.dims:
rechunk_dims[ds.cf.axes["Y"][0]] = rechunk_dims.pop("Y")

ds[rechunk_var] = ds[rechunk_var].chunk(
{d: chnks for d, chnks in rechunk_dims.items() if d in ds[rechunk_var].dims}
)
ds[rechunk_var].encoding.pop("chunksizes", None)
ds[rechunk_var].encoding.pop("chunks", None)

return ds


@parse_config
def rechunk(
path_in: Union[os.PathLike, str, xr.Dataset],
Expand Down
Loading