From 707961b7827f3fbe5b42f89f829e0362cac661f4 Mon Sep 17 00:00:00 2001 From: Greg Lucas Date: Wed, 14 Dec 2022 09:40:47 -0700 Subject: [PATCH] MNT: Use pathlib throughout the codebase Change os.path usages to pathlib.Path() --- docs/make_projection.py | 5 +- docs/source/conf.py | 7 ++- docs/source/matplotlib/advanced_plotting.rst | 9 +--- lib/cartopy/__init__.py | 17 +++--- lib/cartopy/img_transform.py | 8 +-- lib/cartopy/io/__init__.py | 22 ++++---- lib/cartopy/io/img_nest.py | 44 ++++++--------- lib/cartopy/io/img_tiles.py | 27 ++++------ lib/cartopy/io/shapereader.py | 57 +++++++++----------- lib/cartopy/io/srtm.py | 22 ++++---- lib/cartopy/mpl/geoaxes.py | 35 ++++++------ lib/cartopy/tests/io/test_downloaders.py | 26 ++++----- lib/cartopy/tests/mpl/test_images.py | 10 ++-- lib/cartopy/tests/mpl/test_img_transform.py | 5 +- lib/cartopy/tests/test_coding_standards.py | 17 +++--- lib/cartopy/tests/test_img_nest.py | 55 +++++++++---------- lib/cartopy/tests/test_img_tiles.py | 19 ++++--- lib/cartopy/tests/test_shapereader.py | 7 ++- setup.py | 51 ++++++++---------- 19 files changed, 193 insertions(+), 250 deletions(-) diff --git a/docs/make_projection.py b/docs/make_projection.py index 0a503f518..c302ff3e1 100644 --- a/docs/make_projection.py +++ b/docs/make_projection.py @@ -4,8 +4,8 @@ # See COPYING and COPYING.LESSER in the root of the repository for full # licensing details. -import os import inspect +from pathlib import Path import textwrap import numpy as np import cartopy.crs as ccrs @@ -130,8 +130,7 @@ def create_instance(prj_cls, instance_args): if __name__ == '__main__': - fname = os.path.join(os.path.dirname(__file__), 'source', - 'reference', 'projections.rst') + fname = Path(__file__).parent / 'source' / 'reference' / 'projections.rst' table = open(fname, 'w') header = """ diff --git a/docs/source/conf.py b/docs/source/conf.py index 6fe3e9ed5..0df0a8d52 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,7 +17,7 @@ # All configuration values have a default; values that are commented out # serve to show the default. from datetime import datetime -import os +from pathlib import Path import sys import cartopy @@ -25,9 +25,8 @@ from sphinx_gallery.sorting import ExampleTitleSortKey, ExplicitOrder # If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +# add these directories to sys.path here. +sys.path.insert(0, str(Path(__file__).parent.resolve())) # -- General configuration ----------------------------------------------------- diff --git a/docs/source/matplotlib/advanced_plotting.rst b/docs/source/matplotlib/advanced_plotting.rst index e09b1233d..c1cd40397 100644 --- a/docs/source/matplotlib/advanced_plotting.rst +++ b/docs/source/matplotlib/advanced_plotting.rst @@ -12,7 +12,6 @@ Contour plots .. plot:: :include-source: - import os import matplotlib.pyplot as plt from scipy.io import netcdf @@ -21,9 +20,7 @@ Contour plots # get the path of the file. It can be found in the repo data directory. - fname = os.path.join(config["repo_data_dir"], - 'netcdf', 'HadISST1_SST_update.nc' - ) + fname = config["repo_data_dir"] / 'netcdf' / 'HadISST1_SST_update.nc' dataset = netcdf.netcdf_file(fname, maskandscale=True, mmap=False) sst = dataset.variables['sst'][0, :, :] @@ -56,9 +53,7 @@ Images fig = plt.figure(figsize=(8, 12)) # get the path of the file. It can be found in the repo data directory. - fname = os.path.join(config["repo_data_dir"], - 'raster', 'sample', 'Miriam.A2012270.2050.2km.jpg' - ) + fname = config["repo_data_dir"] / 'raster' / 'sample' / 'Miriam.A2012270.2050.2km.jpg' img_extent = (-120.67660000000001, -106.32104523100001, 13.2301484511245, 30.766899999999502) img = plt.imread(fname) diff --git a/lib/cartopy/__init__.py b/lib/cartopy/__init__.py index d2bd7e8c2..96e0b79ae 100644 --- a/lib/cartopy/__init__.py +++ b/lib/cartopy/__init__.py @@ -3,28 +3,27 @@ # This file is part of Cartopy and is released under the LGPL license. # See COPYING and COPYING.LESSER in the root of the repository for full # licensing details. +import os +from pathlib import Path from ._version import version as __version__ # noqa: F401 import tempfile __document_these__ = ['config'] -# Configuration -import os.path - # for the writable data directory (i.e. the one where new data goes), follow # the XDG guidelines found at # https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html -_writable_dir = os.path.join(os.path.expanduser('~'), '.local', 'share') -_data_dir = os.path.join(os.environ.get("XDG_DATA_HOME", _writable_dir), - 'cartopy') -_cache_dir = os.path.join(tempfile.gettempdir(), 'cartopy_cache_dir') +_writable_dir = Path.home() / '.local' / 'share' +_data_dir = Path(os.environ.get("XDG_DATA_HOME", _writable_dir)) / 'cartopy' +_cache_dir = Path(tempfile.gettempdir()) / 'cartopy_cache_dir' -config = {'pre_existing_data_dir': os.environ.get('CARTOPY_DATA_DIR', ''), +config = {'pre_existing_data_dir': Path(os.environ.get('CARTOPY_DATA_DIR', + '')), 'data_dir': _data_dir, 'cache_dir': _cache_dir, - 'repo_data_dir': os.path.join(os.path.dirname(__file__), 'data'), + 'repo_data_dir': Path(__file__).parent / 'data', 'downloaders': {}, } """ diff --git a/lib/cartopy/img_transform.py b/lib/cartopy/img_transform.py index 21c4e7fd2..dbc748817 100644 --- a/lib/cartopy/img_transform.py +++ b/lib/cartopy/img_transform.py @@ -259,12 +259,12 @@ def regrid(array, source_x_coords, source_y_coords, source_proj, target_proj, # Stack our original xyz array, this will also wrap coords when necessary xyz = source_proj.transform_points(source_proj, - source_x_coords.flatten(), - source_y_coords.flatten()) + source_x_coords.flatten(), + source_y_coords.flatten()) # Transform the target points into the source projection target_xyz = source_proj.transform_points(target_proj, - target_x_points.flatten(), - target_y_points.flatten()) + target_x_points.flatten(), + target_y_points.flatten()) if _is_pykdtree: kdtree = pykdtree.kdtree.KDTree(xyz) diff --git a/lib/cartopy/io/__init__.py b/lib/cartopy/io/__init__.py index d00dee4f3..f5cbd8818 100644 --- a/lib/cartopy/io/__init__.py +++ b/lib/cartopy/io/__init__.py @@ -11,7 +11,7 @@ """ import collections -import os +from pathlib import Path import string from urllib.request import urlopen import warnings @@ -150,8 +150,8 @@ def target_path(self, format_dict): expected as a minimum in their ``FORMAT_KEYS`` class attribute. """ - return self._formatter.format(self.target_path_template, - **format_dict) + return Path(self._formatter.format(self.target_path_template, + **format_dict)) def pre_downloaded_path(self, format_dict): """ @@ -167,8 +167,9 @@ def pre_downloaded_path(self, format_dict): expected as a minimum in their ``FORMAT_KEYS`` class attribute. """ - return self._formatter.format(self.pre_downloaded_path_template, - **format_dict) + p = self._formatter.format(self.pre_downloaded_path_template, + **format_dict) + return None if p == '' else Path(p) def path(self, format_dict): """ @@ -193,10 +194,9 @@ def path(self, format_dict): """ pre_downloaded_path = self.pre_downloaded_path(format_dict) target_path = self.target_path(format_dict) - if (pre_downloaded_path is not None and - os.path.exists(pre_downloaded_path)): + if pre_downloaded_path is not None and pre_downloaded_path.exists(): result_path = pre_downloaded_path - elif os.path.exists(target_path): + elif target_path.exists(): result_path = target_path else: # we need to download the file @@ -217,9 +217,9 @@ def acquire_resource(self, target_path, format_dict): expected as a minimum in their ``FORMAT_KEYS`` class attribute. """ - target_dir = os.path.dirname(target_path) - if not os.path.isdir(target_dir): - os.makedirs(target_dir) + target_path = Path(target_path) + target_dir = target_path.parent + target_dir.mkdir(parents=True, exist_ok=True) url = self.url(format_dict) diff --git a/lib/cartopy/io/img_nest.py b/lib/cartopy/io/img_nest.py index c1ca9dfbb..8ad19004f 100644 --- a/lib/cartopy/io/img_nest.py +++ b/lib/cartopy/io/img_nest.py @@ -6,8 +6,7 @@ import collections -import glob -import os.path +from pathlib import Path import numpy as np from PIL import Image @@ -113,32 +112,19 @@ def world_files(fname): '/path/to/img/with_no_extension.w' """ - froot, fext = os.path.splitext(fname) + path = Path(fname) + fext = path.suffix[1::].lower() # If there was no extension to the filename. - if froot == fname: - result = [f'{fname}.w', f'{fname}.W'] + if len(fext) < 3: + result = [path.with_suffix('.w'), path.with_suffix('.W')] else: - fext = fext[1::].lower() - if len(fext) < 3: - result = [f'{fname}.w', f'{fname}.W'] - else: - fext_types = [f'{fext}w', f'{fext[0]}{fext[-1]}w'] - fext_types.extend([ext.upper() for ext in fext_types]) - result = [f'{froot}.{ext}' for ext in fext_types] - - def _convert_basename(name): - dirname, basename = os.path.split(name) - base, ext = os.path.splitext(basename) - if base == base.upper(): - result = base.lower() + ext - else: - result = base.upper() + ext - if dirname: - result = os.path.join(dirname, result) - return result + fext_types = [f'.{fext}w', f'.{fext[0]}{fext[-1]}w'] + fext_types.extend([ext.upper() for ext in fext_types]) + result = [path.with_suffix(ext) for ext in fext_types] - result += [_convert_basename(r) for r in result] - return result + result += [p.with_name(p.stem.swapcase() + p.suffix) for p in result] + # return string paths rather than Path objects + return [str(r) for r in result] def __array__(self): return np.array(Image.open(self.filename)) @@ -228,14 +214,14 @@ def scan_dir_for_imgs(self, directory, glob_pattern='*.tif', Does not recursively search sub-directories. """ - imgs = glob.glob(os.path.join(directory, glob_pattern)) + imgs = Path(directory).glob(glob_pattern) for img in imgs: - dirname, fname = os.path.split(img) + dirname, fname = img.parent, img.name worlds = img_class.world_files(fname) for fworld in worlds: - fworld = os.path.join(dirname, fworld) - if os.path.exists(fworld): + fworld = dirname / fworld + if fworld.exists(): break else: raise ValueError( diff --git a/lib/cartopy/io/img_tiles.py b/lib/cartopy/io/img_tiles.py index fd0ce8ee2..744dde9b1 100644 --- a/lib/cartopy/io/img_tiles.py +++ b/lib/cartopy/io/img_tiles.py @@ -20,7 +20,7 @@ from abc import ABCMeta, abstractmethod import concurrent.futures import io -import os +from pathlib import Path import warnings from PIL import Image @@ -60,11 +60,11 @@ def __init__(self, desired_tile_form='RGB', self._default_cache = False if cache is True: self._default_cache = True - self.cache_path = cartopy.config["cache_dir"] + self.cache_path = Path(cartopy.config["cache_dir"]) elif cache is False: self.cache_path = None else: - self.cache_path = cache + self.cache_path = Path(cache) self.cache = set({}) self._load_cache() @@ -101,22 +101,19 @@ def fetch_tile(tile): @property def _cache_dir(self): """Return the name of the cache directory""" - return os.path.join( - self.cache_path, - self.__class__.__name__ - ) + return self.cache_path / self.__class__.__name__ def _load_cache(self): """Load the cache""" if self.cache_path is not None: cache_dir = self._cache_dir - if not os.path.exists(cache_dir): - os.makedirs(cache_dir) + if not cache_dir.exists(): + cache_dir.mkdir(parents=True) if self._default_cache: warnings.warn( 'Cartopy created the following directory to cache ' f'GoogleWTS tiles: {cache_dir}') - self.cache = self.cache.union(set(os.listdir(cache_dir))) + self.cache = self.cache.union(set(cache_dir.iterdir())) def _find_images(self, target_domain, target_z, start_tile=(0, 0, 0)): """Target domain is a shapely polygon in native coordinates.""" @@ -209,15 +206,11 @@ def get_image(self, tile): if self.cache_path is not None: filename = "_".join([str(i) for i in tile]) + ".npy" - cached_file = os.path.join( - self._cache_dir, - filename - ) + cached_file = self._cache_dir / filename else: - filename = None cached_file = None - if filename in self.cache: + if cached_file in self.cache: img = np.load(cached_file, allow_pickle=False) else: url = self._image_url(tile) @@ -236,7 +229,7 @@ def get_image(self, tile): img = img.convert(self.desired_tile_form) if self.cache_path is not None: np.save(cached_file, img, allow_pickle=False) - self.cache.add(filename) + self.cache.add(cached_file) return img, self.tileextent(tile), 'lower' diff --git a/lib/cartopy/io/shapereader.py b/lib/cartopy/io/shapereader.py index 7398db9bb..4c9663def 100644 --- a/lib/cartopy/io/shapereader.py +++ b/lib/cartopy/io/shapereader.py @@ -27,10 +27,9 @@ """ -import glob import io import itertools -import os +from pathlib import Path from urllib.error import HTTPError import shapely.geometry as sgeom @@ -328,9 +327,8 @@ def acquire_resource(self, target_path, format_dict): """ from zipfile import ZipFile - target_dir = os.path.dirname(target_path) - if not os.path.isdir(target_dir): - os.makedirs(target_dir) + target_dir = Path(target_path).parent + target_dir.mkdir(parents=True, exist_ok=True) url = self.url(format_dict) @@ -339,10 +337,9 @@ def acquire_resource(self, target_path, format_dict): zfh = ZipFile(io.BytesIO(shapefile_online.read()), 'r') for member_path in self.zip_file_contents(format_dict): - ext = os.path.splitext(member_path)[1] - target = os.path.splitext(target_path)[0] + ext - member = zfh.getinfo(member_path.replace(os.sep, '/')) - with open(target, 'wb') as fh: + member = zfh.getinfo(member_path.replace('\\', '/')) + with open(target_path.with_suffix( + Path(member_path).suffix), 'wb') as fh: fh.write(zfh.open(member).read()) shapefile_online.close() @@ -367,9 +364,10 @@ def default_downloader(): """ default_spec = ('shapefiles', 'natural_earth', '{category}', 'ne_{resolution}_{name}.shp') - ne_path_template = os.path.join('{config[data_dir]}', *default_spec) - pre_path_template = os.path.join('{config[pre_existing_data_dir]}', - *default_spec) + ne_path_template = str( + Path('{config[data_dir]}').joinpath(*default_spec)) + pre_path_template = str( + Path('{config[pre_existing_data_dir]}').joinpath(*default_spec)) return NEShpDownloader(target_path_template=ne_path_template, pre_downloaded_path_template=pre_path_template) @@ -429,9 +427,9 @@ def zip_file_contents(self, format_dict): """ for ext in ['.shp', '.dbf', '.shx']: - yield (os.path.join('GSHHS_shp', '{scale}', - 'GSHHS_{scale}_L{level}{extension}' - ).format(extension=ext, **format_dict)) + p = Path('GSHHS_shp', '{scale}', + 'GSHHS_{scale}_L{level}{extension}') + yield str(p).format(extension=ext, **format_dict) def acquire_all_resources(self, format_dict): from zipfile import ZipFile @@ -477,15 +475,13 @@ def acquire_all_resources(self, format_dict): continue modified_format_dict.update({'scale': scale, 'level': level}) target_path = self.target_path(modified_format_dict) - target_dir = os.path.dirname(target_path) - if not os.path.isdir(target_dir): - os.makedirs(target_dir) + target_dir = target_path.parent + target_dir.mkdir(parents=True, exist_ok=True) for member_path in self.zip_file_contents(modified_format_dict): - ext = os.path.splitext(member_path)[1] - target = os.path.splitext(target_path)[0] + ext - member = zfh.getinfo(member_path.replace(os.sep, '/')) - with open(target, 'wb') as fh: + member = zfh.getinfo(member_path.replace('\\', '/')) + with open(target_path.with_suffix( + Path(member_path).suffix), 'wb') as fh: fh.write(zfh.open(member).read()) zfh.close() @@ -502,16 +498,15 @@ def acquire_resource(self, target_path, format_dict): exist in the ``cartopy.config['repo_data_dir']`` directory. """ - repo_fname_pattern = os.path.join(config['repo_data_dir'], - 'shapefiles', 'gshhs', '{scale}', - 'GSHHS_{scale}_L?.shp') + repo_fname_pattern = str(Path('shapefiles') / 'gshhs' + / '{scale}' / 'GSHHS_{scale}_L?.shp') repo_fname_pattern = repo_fname_pattern.format(**format_dict) - repo_fnames = glob.glob(repo_fname_pattern) + repo_fnames = list(config['repo_data_dir'].glob(repo_fname_pattern)) if repo_fnames: assert len(repo_fnames) == 1, '>1 repo files found for GSHHS' return repo_fnames[0] self.acquire_all_resources(format_dict) - if not os.path.exists(target_path): + if not target_path.exists(): raise RuntimeError('Failed to download and extract GSHHS ' f'shapefile to {target_path!r}.') return target_path @@ -535,10 +530,10 @@ def default_downloader(): """ default_spec = ('shapefiles', 'gshhs', '{scale}', 'GSHHS_{scale}_L{level}.shp') - gshhs_path_template = os.path.join('{config[data_dir]}', - *default_spec) - pre_path_tmplt = os.path.join('{config[pre_existing_data_dir]}', - *default_spec) + gshhs_path_template = str( + Path('{config[data_dir]}').joinpath(*default_spec)) + pre_path_tmplt = str( + Path('{config[pre_existing_data_dir]}').joinpath(*default_spec)) return GSHHSShpDownloader(target_path_template=gshhs_path_template, pre_downloaded_path_template=pre_path_tmplt) diff --git a/lib/cartopy/io/srtm.py b/lib/cartopy/io/srtm.py index cfa99b814..8f136c40b 100644 --- a/lib/cartopy/io/srtm.py +++ b/lib/cartopy/io/srtm.py @@ -15,7 +15,7 @@ """ import io -import os +from pathlib import Path import warnings import numpy as np @@ -278,7 +278,7 @@ def read_SRTM(fh): if fname.endswith('.zip'): from zipfile import ZipFile zfh = ZipFile(fh, 'rb') - fh = zfh.open(os.path.basename(fname[:-4]), 'r') + fh = zfh.open(Path(fname).stem, 'r') elev = np.fromfile(fh, dtype=np.dtype('>i2')) if elev.size == 12967201: @@ -289,7 +289,7 @@ def read_SRTM(fh): raise ValueError( f'Shape of SRTM data ({elev.size}) is unexpected.') - fname = os.path.basename(fname) + fname = Path(fname).name y_dir, y, x_dir, x = fname[0], int(fname[1:3]), fname[3], int(fname[4:7]) if y_dir == 'S': @@ -314,8 +314,7 @@ class SRTMDownloader(Downloader): _SRTM_BASE_URL = ('https://e4ftl01.cr.usgs.gov/MEASURES/' 'SRTMGL{resolution}.003/2000.02.11/') - _SRTM_LOOKUP_CACHE = os.path.join(os.path.dirname(__file__), - 'srtm.npz') + _SRTM_LOOKUP_CACHE = Path(__file__).parent / 'srtm.npz' _SRTM_LOOKUP_MASK = np.load(_SRTM_LOOKUP_CACHE)['mask'] """ The SRTM lookup mask determines whether keys such as 'N43E043' are @@ -362,9 +361,8 @@ def url(self, format_dict): def acquire_resource(self, target_path, format_dict): from zipfile import ZipFile - target_dir = os.path.dirname(target_path) - if not os.path.isdir(target_dir): - os.makedirs(target_dir) + target_dir = Path(target_path).parent + target_dir.mkdir(parents=True, exist_ok=True) url = self.url(format_dict) @@ -441,10 +439,10 @@ def default_downloader(cls): """ default_spec = ('SRTM', 'SRTMGL{resolution}', '{y}{x}.hgt') - target_path_template = os.path.join('{config[data_dir]}', - *default_spec) - pre_path_template = os.path.join('{config[pre_existing_data_dir]}', - *default_spec) + target_path_template = str( + Path('{config[data_dir]}').joinpath(*default_spec)) + pre_path_template = str( + Path('{config[pre_existing_data_dir]}').joinpath(*default_spec)) return cls(target_path_template=target_path_template, pre_downloaded_path_template=pre_path_template) diff --git a/lib/cartopy/mpl/geoaxes.py b/lib/cartopy/mpl/geoaxes.py index e7a0c20fe..ed9d82995 100644 --- a/lib/cartopy/mpl/geoaxes.py +++ b/lib/cartopy/mpl/geoaxes.py @@ -15,6 +15,9 @@ import collections import contextlib import functools +import json +import os +from pathlib import Path import warnings import weakref @@ -1010,11 +1013,9 @@ def stock_img(self, name='ne_shaded'): """ if name == 'ne_shaded': - import os source_proj = ccrs.PlateCarree() - fname = os.path.join(config["repo_data_dir"], - 'raster', 'natural_earth', - '50-natural-earth-1-downsampled.png') + fname = (config["repo_data_dir"] / 'raster' / 'natural_earth' + / '50-natural-earth-1-downsampled.png') return self.imshow(imread(fname), origin='upper', transform=source_proj, @@ -1064,11 +1065,9 @@ def background_img(self, name='ne_shaded', resolution='low', extent=None, # read in the user's background image directory: if len(_USER_BG_IMGS) == 0: self.read_user_background_images() - import os - bgdir = os.getenv('CARTOPY_USER_BACKGROUNDS') - if bgdir is None: - bgdir = os.path.join(config["repo_data_dir"], - 'raster', 'natural_earth') + bgdir = Path(os.getenv( + 'CARTOPY_USER_BACKGROUNDS', + config["repo_data_dir"] / 'raster' / 'natural_earth')) # now get the filename we want to use: try: fname = _USER_BG_IMGS[name][resolution] @@ -1078,7 +1077,7 @@ def background_img(self, name='ne_shaded', resolution='low', extent=None, f'present in the user background image metadata in directory ' f'{bgdir!r}') # Now obtain the image data from file or cache: - fpath = os.path.join(bgdir, fname) + fpath = bgdir / fname if cache: if fname in _BACKG_IMG_CACHE: img = _BACKG_IMG_CACHE[fname] @@ -1164,14 +1163,10 @@ def read_user_background_images(self, verify=True): lib/cartopy/data/raster/natural_earth/images.json """ - import os - import json - - bgdir = os.getenv('CARTOPY_USER_BACKGROUNDS') - if bgdir is None: - bgdir = os.path.join(config["repo_data_dir"], - 'raster', 'natural_earth') - json_file = os.path.join(bgdir, 'images.json') + bgdir = Path(os.getenv( + 'CARTOPY_USER_BACKGROUNDS', + config["repo_data_dir"] / 'raster' / 'natural_earth')) + json_file = bgdir / 'images.json' with open(json_file) as js_obj: dict_in = json.load(js_obj) @@ -1196,8 +1191,8 @@ def read_user_background_images(self, verify=True): # the required_info items are not resolutions: if resln not in required_info: img_it_r = _USER_BG_IMGS[img_type][resln] - test_file = os.path.join(bgdir, img_it_r) - if not os.path.isfile(test_file): + test_file = bgdir / img_it_r + if not test_file.is_file(): raise ValueError( f'File "{test_file}" not found') diff --git a/lib/cartopy/tests/io/test_downloaders.py b/lib/cartopy/tests/io/test_downloaders.py index ba4b65b72..c9f979394 100644 --- a/lib/cartopy/tests/io/test_downloaders.py +++ b/lib/cartopy/tests/io/test_downloaders.py @@ -5,7 +5,7 @@ # licensing details. import contextlib -import os +from pathlib import Path from unittest import mock import pytest @@ -17,21 +17,21 @@ def test_Downloader_data(): di = cio.Downloader('https://testing.com/{category}/{name}.zip', - os.path.join('{data_dir}', '{category}', - 'shape.shp'), - '/project/foobar/{category}/sample.shp') + str(Path('{data_dir}') / '{category}' / 'shape.shp'), + str(Path('/project') / 'foobar' / '{category}' / + 'sample.shp')) replacement_dict = {'category': 'example', 'name': 'test', - 'data_dir': os.path.join('/wibble', 'foo', 'bar')} + 'data_dir': str(Path('/wibble') / 'foo' / 'bar')} assert di.url(replacement_dict) == 'https://testing.com/example/test.zip' assert (di.target_path(replacement_dict) == - os.path.join('/wibble', 'foo', 'bar', 'example', 'shape.shp')) + Path('/wibble') / 'foo' / 'bar' / 'example' / 'shape.shp') assert (di.pre_downloaded_path(replacement_dict) == - '/project/foobar/example/sample.shp') + Path('/project/foobar/example/sample.shp')) @contextlib.contextmanager @@ -105,7 +105,7 @@ def test_downloading_simple_ascii(download_to_temp): format_dict = {'name': 'jquery'} target_template = str(download_to_temp / '{name}.txt') - tmp_fname = target_template.format(**format_dict) + tmp_fname = Path(target_template.format(**format_dict)) dnld_item = cio.Downloader(file_url, target_template) @@ -151,7 +151,7 @@ def test_natural_earth_downloader(tmp_path): shp_path = dnld_item.path(format_dict) counter.assert_called_once() - assert shp_path_template.format(**format_dict) == shp_path + assert shp_path_template.format(**format_dict) == str(shp_path) # check that calling path again doesn't try re-downloading with mock.patch.object(dnld_item, 'acquire_resource', @@ -162,13 +162,13 @@ def test_natural_earth_downloader(tmp_path): # check that we have the shp and the shx exts = ['.shp', '.shx'] for ext in exts: - stem = os.path.splitext(shp_path)[0] - assert os.path.exists(stem + ext), \ - f"Shapefile's {ext} file doesn't exist in {stem}{ext}" + fname = shp_path.with_suffix(ext) + assert fname.exists(), \ + f"Shapefile's {ext} file doesn't exist in {fname}" # check that providing a pre downloaded path actually works pre_dnld = NEShpDownloader(target_path_template='/not/a/real/file.txt', - pre_downloaded_path_template=shp_path) + pre_downloaded_path_template=str(shp_path)) # check that the pre_dnld downloader doesn't re-download, but instead # uses the path of the previously downloaded item diff --git a/lib/cartopy/tests/mpl/test_images.py b/lib/cartopy/tests/mpl/test_images.py index d04972aa7..d7e18d3fb 100644 --- a/lib/cartopy/tests/mpl/test_images.py +++ b/lib/cartopy/tests/mpl/test_images.py @@ -4,7 +4,6 @@ # See COPYING and COPYING.LESSER in the root of the repository for full # licensing details. -import os import types import numpy as np @@ -21,11 +20,10 @@ import cartopy.tests.test_img_tiles as ctest_tiles -NATURAL_EARTH_IMG = os.path.join(config["repo_data_dir"], - 'raster', 'natural_earth', - '50-natural-earth-1-downsampled.png') -REGIONAL_IMG = os.path.join(config['repo_data_dir'], 'raster', 'sample', - 'Miriam.A2012270.2050.2km.jpg') +NATURAL_EARTH_IMG = (config["repo_data_dir"] / 'raster' / 'natural_earth' + / '50-natural-earth-1-downsampled.png') +REGIONAL_IMG = (config['repo_data_dir'] / 'raster' / 'sample' + / 'Miriam.A2012270.2050.2km.jpg') # We have an exceptionally large tolerance for the web_tiles test. diff --git a/lib/cartopy/tests/mpl/test_img_transform.py b/lib/cartopy/tests/mpl/test_img_transform.py index fb3f38810..31dd0fab9 100644 --- a/lib/cartopy/tests/mpl/test_img_transform.py +++ b/lib/cartopy/tests/mpl/test_img_transform.py @@ -5,7 +5,6 @@ # licensing details. import operator -import os import matplotlib as mpl import matplotlib.pyplot as plt @@ -82,8 +81,8 @@ def test_different_dims(self): @pytest.mark.mpl_image_compare(filename='regrid_image.png', tolerance=5.55) def test_regrid_image(): # Source data - fname = os.path.join(config["repo_data_dir"], 'raster', 'natural_earth', - '50-natural-earth-1-downsampled.png') + fname = (config["repo_data_dir"] / 'raster' / 'natural_earth' + / '50-natural-earth-1-downsampled.png') nx = 720 ny = 360 source_proj = ccrs.PlateCarree() diff --git a/lib/cartopy/tests/test_coding_standards.py b/lib/cartopy/tests/test_coding_standards.py index 0d89f2276..3eb22da3e 100644 --- a/lib/cartopy/tests/test_coding_standards.py +++ b/lib/cartopy/tests/test_coding_standards.py @@ -5,6 +5,7 @@ # licensing details. from fnmatch import fnmatch +from pathlib import Path import os import re import subprocess @@ -31,9 +32,9 @@ # Guess cartopy repo directory of cartopy - realpath is used to mitigate # against Python finding the cartopy package via a symlink. -CARTOPY_DIR = os.path.realpath(os.path.dirname(cartopy.__file__)) -REPO_DIR = os.getenv('CARTOPY_GIT_DIR', - os.path.dirname(os.path.dirname(CARTOPY_DIR))) +CARTOPY_DIR = Path(cartopy.__file__).parent.resolve() +REPO_DIR = Path(os.getenv('CARTOPY_GIT_DIR', + CARTOPY_DIR.parent.parent)) class TestLicenseHeaders: @@ -50,7 +51,7 @@ def list_tracked_files(): """ # Check the ".git" folder exists at the repo dir. - if not os.path.isdir(os.path.join(REPO_DIR, '.git')): + if not (REPO_DIR / '.git').is_dir(): raise ValueError(f'{REPO_DIR} is not a git repository.') output = subprocess.check_output(['git', 'ls-tree', '-z', '-r', @@ -79,13 +80,13 @@ def test_license_headers(self): failed = [] for fname in sorted(tracked_files): - full_fname = os.path.join(REPO_DIR, fname) - root, ext = os.path.splitext(full_fname) + full_fname = REPO_DIR / fname + ext = full_fname.suffix if ext in ('.py', '.pyx', '.c', '.cpp', '.h') and \ - os.path.isfile(full_fname) and \ + full_fname.is_file() and \ not any(fnmatch(fname, pat) for pat in exclude_patterns): - if os.path.getsize(full_fname) == 0: + if full_fname.stat().st_size == 0: # Allow completely empty files (e.g. ``__init__.py``) continue diff --git a/lib/cartopy/tests/test_img_nest.py b/lib/cartopy/tests/test_img_nest.py index 8099f63f4..685c2cb5c 100644 --- a/lib/cartopy/tests/test_img_nest.py +++ b/lib/cartopy/tests/test_img_nest.py @@ -5,7 +5,7 @@ # licensing details. import io -import os +from pathlib import Path import pickle import shutil import sys @@ -25,8 +25,7 @@ #: An integer version which should be increased if the test data needs #: to change in some way. _TEST_DATA_VERSION = 1 -_TEST_DATA_DIR = os.path.join(config["data_dir"], - 'wmts', 'aerial') +_TEST_DATA_DIR = config["data_dir"] / 'wmts' / 'aerial' @pytest.mark.parametrize('fname, expected', [ @@ -124,9 +123,9 @@ def test_intersect(tmp_path): im.save(z_2_dir / 'p2-5.tif') # Provided in reverse order in order to test the area sorting. - items = [('dummy-z-2', str(z_2_dir)), - ('dummy-z-1', str(z_1_dir)), - ('dummy-z-0', str(z_0_dir))] + items = [('dummy-z-2', z_2_dir), + ('dummy-z-1', z_1_dir), + ('dummy-z-0', z_0_dir)] nic = cimg_nest.NestedImageCollection.from_configuration('dummy', None, items) @@ -149,8 +148,7 @@ def test_intersect(tmp_path): for zoom, image_names in expected: key = [k for k in nic._ancestry.keys() if k[0] == zoom][0] ancestry = nic._ancestry[key] - fnames = sorted(os.path.basename(item[1].filename) - for item in ancestry) + fnames = sorted(item[1].filename.name for item in ancestry) assert image_names == fnames # Check image retrieval for specific domain. @@ -170,9 +168,8 @@ def _tile_from_img(img): like "lib/cartopy/data/wmts/aerial/z_0/x_0_y0.png" """ - _, z = os.path.basename(os.path.dirname(img.filename)).split('_') - xy, _ = os.path.splitext(os.path.basename(img.filename)) - _, x, _, y = xy.split('_') + _, z = img.filename.parent.name.split('_') + _, x, _, y = img.filename.stem.split('_') return int(x), int(y), int(z) @@ -195,15 +192,15 @@ def world_file_extent(*args, **kwargs): def test_nest(nest_from_config): crs = cimgt.GoogleTiles().crs z0 = cimg_nest.ImageCollection('aerial z0 test', crs) - z0.scan_dir_for_imgs(os.path.join(_TEST_DATA_DIR, 'z_0'), + z0.scan_dir_for_imgs(_TEST_DATA_DIR / 'z_0', glob_pattern='*.png', img_class=RoundedImg) z1 = cimg_nest.ImageCollection('aerial z1 test', crs) - z1.scan_dir_for_imgs(os.path.join(_TEST_DATA_DIR, 'z_1'), + z1.scan_dir_for_imgs(_TEST_DATA_DIR / 'z_1', glob_pattern='*.png', img_class=RoundedImg) z2 = cimg_nest.ImageCollection('aerial z2 test', crs) - z2.scan_dir_for_imgs(os.path.join(_TEST_DATA_DIR, 'z_2'), + z2.scan_dir_for_imgs(_TEST_DATA_DIR / 'z_2', glob_pattern='*.png', img_class=RoundedImg) # make sure all the images from z1 are contained by the z0 image. The @@ -234,7 +231,7 @@ def test_nest(nest_from_config): assert ('aerial z1 test', img) in nest_z0_z1._ancestry[key] x1_y0_z1, = (img for img in z1.images - if img.filename.endswith('x_1_y_0.png')) + if img.filename.name.endswith('x_1_y_0.png')) assert (1, 0, 1) == _tile_from_img(x1_y0_z1) @@ -284,12 +281,11 @@ def wmts_data(): for sub_tile in aerial.subtiles(tile): tiles.append(sub_tile) - fname_template = os.path.join(_TEST_DATA_DIR, 'z_{}', 'x_{}_y_{}.png') + fname_template = str(_TEST_DATA_DIR / 'z_{}' / 'x_{}_y_{}.png') - if not os.path.isdir(_TEST_DATA_DIR): - os.makedirs(_TEST_DATA_DIR) + _TEST_DATA_DIR.mkdir(parents=True, exist_ok=True) - data_version_fname = os.path.join(_TEST_DATA_DIR, 'version.txt') + data_version_fname = _TEST_DATA_DIR / 'version.txt' test_data_version = None try: @@ -302,17 +298,16 @@ def wmts_data(): warnings.warn('WMTS test data is out of date, regenerating at ' f'{_TEST_DATA_DIR}.') shutil.rmtree(_TEST_DATA_DIR) - os.makedirs(_TEST_DATA_DIR) + _TEST_DATA_DIR.mkdir(parents=True) with open(data_version_fname, 'w') as fh: fh.write(str(_TEST_DATA_VERSION)) # Download the tiles. for tile in tiles: x, y, z = tile - fname = fname_template.format(z, x, y) - if not os.path.exists(fname): - if not os.path.isdir(os.path.dirname(fname)): - os.makedirs(os.path.dirname(fname)) + fname = Path(fname_template.format(z, x, y)) + if not fname.exists(): + fname.parent.mkdir(parents=True, exist_ok=True) img, extent, _ = aerial.get_image(tile) nx, ny = 256, 256 @@ -325,7 +320,7 @@ def wmts_data(): upper_left_center = (extent[0] + pix_size_x / 2, extent[2] + pix_size_y / 2) - pgw_fname = fname[:-4] + '.pgw' + pgw_fname = fname.with_suffix('.pgw') pgw_keys = {'x_pix_size': np.float64(pix_size_x), 'y_rotation': 0, 'x_rotation': 0, @@ -339,9 +334,9 @@ def wmts_data(): @pytest.mark.network def test_find_images(wmts_data): - z2_dir = os.path.join(_TEST_DATA_DIR, 'z_2') - img_fname = os.path.join(z2_dir, 'x_2_y_0.png') - world_file_fname = os.path.join(z2_dir, 'x_2_y_0.pgw') + z2_dir = _TEST_DATA_DIR / 'z_2' + img_fname = z2_dir / 'x_2_y_0.png' + world_file_fname = z2_dir / 'x_2_y_0.pgw' img = RoundedImg.from_world_file(img_fname, world_file_fname) assert img.filename == img_fname @@ -358,8 +353,8 @@ def test_find_images(wmts_data): def nest_from_config(wmts_data): from_config = cimg_nest.NestedImageCollection.from_configuration - files = [['aerial z0 test', os.path.join(_TEST_DATA_DIR, 'z_0')], - ['aerial z1 test', os.path.join(_TEST_DATA_DIR, 'z_1')], + files = [['aerial z0 test', _TEST_DATA_DIR / 'z_0'], + ['aerial z1 test', _TEST_DATA_DIR / 'z_1'], ] crs = cimgt.GoogleTiles().crs diff --git a/lib/cartopy/tests/test_img_tiles.py b/lib/cartopy/tests/test_img_tiles.py index ec18c9950..84069c958 100644 --- a/lib/cartopy/tests/test_img_tiles.py +++ b/lib/cartopy/tests/test_img_tiles.py @@ -227,15 +227,15 @@ def test_ordnance_survey_tile_styles(): tile = ["1", "2", "3"] # Default is Road_3857. - os = cimgt.OrdnanceSurvey(dummy_apikey) - url = os._image_url(tile) + ordsurvey = cimgt.OrdnanceSurvey(dummy_apikey) + url = ordsurvey._image_url(tile) assert url == ref_url.format(layer="Road_3857", z=tile[2], y=tile[1], x=tile[0]) for layer in ("Road_3857", "Light_3857", "Outdoor_3857", "Road", "Light", "Outdoor"): - os = cimgt.OrdnanceSurvey(dummy_apikey, layer=layer) - url = os._image_url(tile) + ordsurvey = cimgt.OrdnanceSurvey(dummy_apikey, layer=layer) + url = ordsurvey._image_url(tile) layer = layer if layer.endswith("_3857") else layer + "_3857" assert url == ref_url.format(layer=layer, z=tile[2], y=tile[1], x=tile[0]) @@ -362,17 +362,16 @@ def test_cache(cache_dir, tmp_path): ] # Check the results - cache_dir_res = os.path.join(gt.cache_path, "GoogleTiles") - files = [i for i in os.listdir(cache_dir_res)] + cache_dir_res = gt.cache_path / "GoogleTiles" + files = list(cache_dir_res.iterdir()) hashes = { f: hashlib.md5( - np.load(os.path.join(cache_dir_res, f), allow_pickle=True).data + np.load(cache_dir_res / f, allow_pickle=True).data ).hexdigest() for f in files } - - assert sorted(files) == [f for x, y, z, f, h in x_y_z_f_h] + assert sorted(files) == [cache_dir_res / f for x, y, z, f, h in x_y_z_f_h] assert set(files) == gt.cache assert sorted(hashes.values()) == sorted( @@ -381,7 +380,7 @@ def test_cache(cache_dir, tmp_path): # Update images in cache (all white) for f in files: - filename = os.path.join(cache_dir_res, f) + filename = cache_dir_res / f img = np.load(filename, allow_pickle=True) img.fill(255) np.save(filename, img, allow_pickle=True) diff --git a/lib/cartopy/tests/test_shapereader.py b/lib/cartopy/tests/test_shapereader.py index 5539cae67..94a9401a4 100644 --- a/lib/cartopy/tests/test_shapereader.py +++ b/lib/cartopy/tests/test_shapereader.py @@ -3,8 +3,7 @@ # This file is part of Cartopy and is released under the LGPL license. # See COPYING and COPYING.LESSER in the root of the repository for full # licensing details. - -import os.path +from pathlib import Path import numpy as np from numpy.testing import assert_array_almost_equal @@ -16,8 +15,8 @@ class TestLakes: def setup_class(self): - LAKES_PATH = os.path.join(os.path.dirname(__file__), - 'lakes_shapefile', 'ne_110m_lakes.shp') + LAKES_PATH = (Path(__file__).parent / 'lakes_shapefile' + / 'ne_110m_lakes.shp') self.reader = shp.Reader(LAKES_PATH) names = [record.attributes['name'] for record in self.reader.records()] # Choose a nice small lake diff --git a/setup.py b/setup.py index 04c357a40..5101b0705 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ sys.exit(error) +from pathlib import Path import os import subprocess import warnings @@ -39,8 +40,8 @@ # The existence of a PKG-INFO directory is enough to tell us whether this is a # source installation or not (sdist). -HERE = os.path.dirname(__file__) -IS_SDIST = os.path.exists(os.path.join(HERE, 'PKG-INFO')) +HERE = Path(__file__).parent +IS_SDIST = (HERE / 'PKG-INFO').exists() FORCE_CYTHON = os.environ.get('FORCE_CYTHON', False) if not IS_SDIST or FORCE_CYTHON: @@ -62,17 +63,14 @@ GEOS_MIN_VERSION = (3, 7, 2) -def file_walk_relative(top, remove=''): +def file_walk_relative(root): """ - Return a generator of files from the top of the tree, removing - the given prefix from the root/file result. + Return a list of files from the top of the tree, removing + the lib/cartopy prefix from the resulting paths. """ - top = top.replace('/', os.path.sep) - remove = remove.replace('/', os.path.sep) - for root, dirs, files in os.walk(top): - for file in files: - yield os.path.join(root, file).replace(remove, '') + return [str(p.relative_to(Path('lib') / 'cartopy')) + for p in root.glob("**/*")] # Dependency checks @@ -114,9 +112,9 @@ def file_walk_relative(top, remove=''): # Python dependencies extras_require = {} -for name in os.listdir(os.path.join(HERE, 'requirements')): - with open(os.path.join(HERE, 'requirements', name)) as fh: - section, ext = os.path.splitext(name) +for name in (HERE / 'requirements').iterdir(): + with open(name) as fh: + section, ext = name.stem, name.suffix extras_require[section] = [] for line in fh: if line.startswith('#'): @@ -142,7 +140,7 @@ def get_config_var(name): # Description # =========== -with open(os.path.join(HERE, 'README.md')) as fh: +with open(HERE / 'README.md') as fh: description = ''.join(fh.readlines()) @@ -181,13 +179,14 @@ def decythonize(extensions, **_ignore): for extension in extensions: sources = [] for sfile in extension.sources: - path, ext = os.path.splitext(sfile) + spath = Path(sfile) + ext = spath.suffix if ext in ('.pyx',): if extension.language == 'c++': ext = '.cpp' else: ext = '.c' - sfile = path + ext + sfile = str(spath.with_suffix(ext)) sources.append(sfile) extension.sources[:] = sources return extensions @@ -199,6 +198,12 @@ def decythonize(extensions, **_ignore): else: cmdclass = {'build_ext': cy_build_ext} +base_path = Path('lib') / 'cartopy' +package_data = ( + file_walk_relative(base_path / 'tests' / 'mpl' / 'baseline_images') + + file_walk_relative(base_path / 'data') + + file_walk_relative(base_path / 'tests' / 'lakes_shapefile') + + ['io/srtm.npz']) # Main setup # ========== @@ -225,19 +230,7 @@ def decythonize(extensions, **_ignore): packages=find_packages("lib"), package_dir={'': 'lib'}, - package_data={'cartopy': list(file_walk_relative('lib/cartopy/tests/' - 'mpl/baseline_images/', - remove='lib/cartopy/')) + - list(file_walk_relative('lib/cartopy/data/raster', - remove='lib/cartopy/')) + - list(file_walk_relative('lib/cartopy/data/netcdf', - remove='lib/cartopy/')) + - list(file_walk_relative('lib/cartopy/data/' - 'shapefiles/gshhs', - remove='lib/cartopy/')) + - list(file_walk_relative('lib/cartopy/tests/lakes_shapefile', - remove='lib/cartopy/')) + - ['io/srtm.npz']}, + package_data={'cartopy': package_data}, scripts=['tools/cartopy_feature_download.py'], ext_modules=extensions,