fix: podaac now has different openers for s3 and data endpoints (#120)

* feat: add piecewise program
* feat: add additional fit term options to regression programs
* docs: remove binder links :(
* fix: add podaac cumulus test and skip drive
* fix: bump GFZ GravIS files to Release-03

README.rst

@@ -6,8 +6,6 @@ gravity-toolkit
 |License|
 |PyPI Version|
 |Documentation Status|
-|Binder|
-|Pangeo|
 |zenodo|
 
 .. |Language| image:: https://img.shields.io/pypi/pyversions/gravity-toolkit?color=green
@@ -22,12 +20,6 @@ gravity-toolkit
 .. |Documentation Status| image:: https://readthedocs.org/projects/gravity-toolkit/badge/?version=latest
    :target: https://gravity-toolkit.readthedocs.io/en/latest/?badge=latest
 
-.. |Binder| image:: https://mybinder.org/badge_logo.svg
-   :target: https://mybinder.org/v2/gh/tsutterley/gravity-toolkit/main
-
-.. |Pangeo| image:: https://img.shields.io/static/v1.svg?logo=Jupyter&label=PangeoBinderAWS&message=us-west-2&color=orange
-   :target: https://aws-uswest2-binder.pangeo.io/v2/gh/tsutterley/gravity-toolkit/main?urlpath=lab
-
 .. |zenodo| image:: https://zenodo.org/badge/107323776.svg
    :target: https://zenodo.org/badge/latestdoi/107323776
 

doc/source/api_reference/piecewise_grace_maps.rst

@@ -0,0 +1,35 @@
+=======================
+piecewise_grace_maps.py
+=======================
+
+- Reads in GRACE/GRACE-FO spatial files and fits a piecewise regression model at each grid point for breakpoint analysis
+
+`Source code`__
+
+.. __: https://github.com/tsutterley/gravity-toolkit/blob/main/scripts/piecewise_grace_maps.py
+
+Calling Sequence
+################
+
+.. argparse::
+    :filename: piecewise_grace_maps.py
+    :func: arguments
+    :prog: piecewise_grace_maps.py
+    :nodescription:
+    :nodefault:
+
+    --units -U : @after
+        * ``1``: cm of water thickness
+        * ``2``: mm of geoid height
+        * ``3``: mm of elastic crustal deformation
+        * ``4``: |mu|\ Gal gravitational perturbation
+        * ``5``: mbar equivalent surface pressure
+
+    --interval : @replace
+        Output grid interval
+
+        * ``1``: (0:360, 90:-90)
+        * ``2``: (degree spacing/2)
+        * ``3``: non-global grid (set with defined bounds)
+
+.. |mu|      unicode:: U+03BC .. GREEK SMALL LETTER MU

doc/source/getting_started/Install.rst

@@ -29,12 +29,3 @@ Alternatively can install the ``gravity_toolkit`` utilities directly from GitHub
 .. code-block:: bash
 
     python3 -m pip install --user git+https://github.com/tsutterley/gravity-toolkit.git
-
-| This repository can be also tested using `BinderHub <https://github.com/jupyterhub/binderhub>`_ platforms:
-| |Binder| |Pangeo|
-
-.. |Binder| image:: https://mybinder.org/badge_logo.svg
-   :target: https://mybinder.org/v2/gh/tsutterley/gravity-toolkit/main
-
-.. |Pangeo| image:: https://img.shields.io/static/v1.svg?logo=Jupyter&label=PangeoBinderAWS&message=us-west-2&color=orange
-   :target: https://aws-uswest2-binder.pangeo.io/v2/gh/tsutterley/gravity-toolkit/main?urlpath=lab

doc/source/index.rst

@@ -118,6 +118,7 @@ missions
     api_reference/mascon_reconstruct.rst
     api_reference/monte_carlo_degree_one.rst
     api_reference/quick_mascon_regress.rst
+    api_reference/piecewise_grace_maps.rst
     api_reference/regress_grace_maps.rst
     api_reference/run_grace_date.rst
     api_reference/run_sea_level_equation.rst

gravity_toolkit/SLR/C20.py

@@ -256,7 +256,7 @@ def C20(SLR_file, AOD=True, HEADER=True):
         n_mon = file_lines - count
         # date and GRACE/GRACE-FO month
         dinput['time'] = np.zeros((n_mon))
-        dinput['month'] = np.zeros((n_mon),dtype=int)
+        dinput['month'] = np.zeros((n_mon), dtype=int)
         # monthly spherical harmonic replacement solutions
         dinput['data'] = np.zeros((n_mon))
         # monthly spherical harmonic formal standard deviations

gravity_toolkit/SLR/C30.py

@@ -140,7 +140,7 @@ def C30(SLR_file, C30_MEAN=9.5717395773300e-07, HEADER=True):
         n_mon = file_lines - count
         # date and GRACE/GRACE-FO month
         dinput['time'] = np.zeros((n_mon))
-        dinput['month'] = np.zeros((n_mon),dtype=int)
+        dinput['month'] = np.zeros((n_mon), dtype=int)
         # monthly spherical harmonic replacement solutions
         dinput['data'] = np.zeros((n_mon))
         # monthly spherical harmonic formal standard deviations
@@ -215,7 +215,7 @@ def C30(SLR_file, C30_MEAN=9.5717395773300e-07, HEADER=True):
         n_mon = file_lines - count
         # date and GRACE/GRACE-FO month
         dinput['time'] = np.zeros((n_mon))
-        dinput['month'] = np.zeros((n_mon),dtype=int)
+        dinput['month'] = np.zeros((n_mon), dtype=int)
         # monthly spherical harmonic replacement solutions
         dinput['data'] = np.zeros((n_mon))
         # monthly spherical harmonic formal standard deviations

gravity_toolkit/SLR/C50.py

@@ -119,7 +119,7 @@ def C50(SLR_file, C50_MEAN=0.0, DATE=None, HEADER=True):
         n_mon = file_lines - count
         # date and GRACE/GRACE-FO month
         dinput['time'] = np.zeros((n_mon))
-        dinput['month'] = np.zeros((n_mon),dtype=int)
+        dinput['month'] = np.zeros((n_mon), dtype=int)
         # monthly spherical harmonic replacement solutions
         dinput['data'] = np.zeros((n_mon))
         # monthly spherical harmonic formal standard deviations

gravity_toolkit/SLR/CS2.py

@@ -236,7 +236,7 @@ def CS2(SLR_file, ORDER=1, DATE=None, HEADER=True):
         n_mon = file_lines - count
         # date and GRACE/GRACE-FO month
         dinput['time'] = np.zeros((n_mon))
-        dinput['month'] = np.zeros((n_mon),dtype=int)
+        dinput['month'] = np.zeros((n_mon), dtype=int)
         # monthly spherical harmonic replacement solutions
         dinput['C2m'] = np.zeros((n_mon))
         dinput['S2m'] = np.zeros((n_mon))

gravity_toolkit/geocenter.py

@@ -250,7 +250,7 @@ def from_gravis(self, geocenter_file, **kwargs):
         n_mon = file_lines - count
         # date and GRACE/GRACE-FO month
         dinput['time'] = np.zeros((n_mon))
-        dinput['month'] = np.zeros((n_mon),dtype=int)
+        dinput['month'] = np.zeros((n_mon), dtype=int)
         # monthly spherical harmonic replacement solutions
         dinput['C10'] = np.zeros((n_mon))
         dinput['C11'] = np.zeros((n_mon))

gravity_toolkit/grace_input_months.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 u"""
 grace_input_months.py
-Written by Tyler Sutterley (03/2023)
+Written by Tyler Sutterley (04/2023)
 Contributions by Hugo Lecomte and Yara Mohajerani
 
 Reads GRACE/GRACE-FO files for a specified spherical harmonic degree and order
@@ -109,6 +109,7 @@
     read_gfc_harmonics.py: reads spherical harmonic data from gfc files
 
 UPDATE HISTORY:
+    Updated 04/2023: use release-03 GFZ GravIS SLR and geocenter files
     Updated 03/2023: added attributes for input files and corrections
         improve typing for variables in docstrings
     Updated 01/2023: refactored satellite laser ranging read functions
@@ -477,7 +478,7 @@ def grace_input_months(base_dir, PROC, DREL, DSET, LMAX, start_mon, end_mon,
         FLAGS.append('_wCSR_21')
         attributes['SLR 21'] = ('CSR',os.path.basename(SLR_file))
     elif (kwargs['SLR_21'] == 'GFZ'):
-        GravIS_file = 'GRAVIS-2B_GFZOP_GRACE+SLR_LOW_DEGREES_0002.dat'
+        GravIS_file = 'GRAVIS-2B_GFZOP_GRACE+SLR_LOW_DEGREES_0003.dat'
         SLR_file = os.path.join(base_dir,GravIS_file)
         # log SLR file if debugging
         logging.debug(f'Reading SLR C21/S21 file: {SLR_file}')
@@ -533,7 +534,7 @@ def grace_input_months(base_dir, PROC, DREL, DSET, LMAX, start_mon, end_mon,
         FLAGS.append('_wLARES_C30')
         attributes['SLR_C30'] = ('CSR LARES',os.path.basename(SLR_file))
     elif (kwargs['SLR_C30'] == 'GFZ'):
-        GravIS_file = 'GRAVIS-2B_GFZOP_GRACE+SLR_LOW_DEGREES_0002.dat'
+        GravIS_file = 'GRAVIS-2B_GFZOP_GRACE+SLR_LOW_DEGREES_0003.dat'
         SLR_file = os.path.join(base_dir,GravIS_file)
         # log SLR file if debugging
         logging.debug(f'Reading SLR C30 file: {SLR_file}')
@@ -684,7 +685,7 @@ def grace_input_months(base_dir, PROC, DREL, DSET, LMAX, start_mon, end_mon,
         # degree 1 coefficients provided by GFZ GravIS
         # http://gravis.gfz-potsdam.de/corrections
         default_geocenter = os.path.join(base_dir,'geocenter',
-            'GRAVIS-2B_GFZOP_GEOCENTER_0002.dat')
+            'GRAVIS-2B_GFZOP_GEOCENTER_0003.dat')
         # read degree one files from GFZ GravIS
         DEG1_file = kwargs.get('DEG1_FILE') or default_geocenter
         # log geocenter file if debugging

gravity_toolkit/time_series/piecewise.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 u"""
 piecewise.py
-Written by Tyler Sutterley (01/2023)
+Written by Tyler Sutterley (04/2023)
 
 Fits a synthetic signal to data over a time period by ordinary or weighted
     least-squares for breakpoint analysis
@@ -51,6 +51,7 @@
         array if unequal for weighted least squares
     WEIGHT: Set if measurement errors for use in weighted least squares
     CYCLES: list of cyclical terms (0.5=semi-annual, 1=annual)
+    TERMS: list of extra terms
     STDEV: standard deviation of output error
     CONF: confidence interval of output error
     AICc: use second order AIC
@@ -60,6 +61,7 @@
     scipy: Scientific Tools for Python (https://docs.scipy.org/doc/)
 
 UPDATE HISTORY:
+    Updated 04/2023: option to include extra fit terms in the design matrix
     Updated 01/2023: refactored time series analysis functions
     Updated 04/2022: updated docstrings to numpy documentation format
     Updated 05/2021: define int/float precision to prevent deprecation warning
@@ -95,8 +97,8 @@
 import scipy.special
 
 def piecewise(t_in, d_in, BREAK_TIME=None, BREAKPOINT=None,
-    CYCLES=[0.5,1.0], DATA_ERR=0, WEIGHT=False, STDEV=0, CONF=0,
-    AICc=False):
+    CYCLES=[0.5,1.0], TERMS=[], DATA_ERR=0, WEIGHT=False,
+    STDEV=0, CONF=0, AICc=False):
     """
     Fits a synthetic signal to data over a time period by ordinary or
     weighted least-squares for breakpoint analysis [Toms2003]_
@@ -113,6 +115,8 @@ def piecewise(t_in, d_in, BREAK_TIME=None, BREAKPOINT=None,
         breakpoint indice of piecewise regression
     CYCLES: list, default [0.5, 1.0]
         list of cyclical terms in fractions of year
+    TERMS: list, default []
+        list of extra fit terms
     DATA_ERR: float or list
         data precision
 
@@ -205,6 +209,9 @@ def piecewise(t_in, d_in, BREAK_TIME=None, BREAKPOINT=None,
     for c in CYCLES:
         DMAT.append(np.sin(2.0*np.pi*t_in/np.float64(c)))
         DMAT.append(np.cos(2.0*np.pi*t_in/np.float64(c)))
+    # add additional terms to the design matrix
+    for t in TERMS:
+        DMAT.append(t)
     # take the transpose of the design matrix
     DMAT = np.transpose(DMAT)
 

gravity_toolkit/time_series/regress.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 u"""
 regress.py
-Written by Tyler Sutterley (01/2023)
+Written by Tyler Sutterley (04/2023)
 
 Fits a synthetic signal to data over a time period by ordinary or weighted
     least-squares
@@ -46,6 +46,7 @@
     RELATIVE: relative period
     ORDER: maximum polynomial order in fit (0=constant, 1=linear, 2=quadratic)
     CYCLES: list of cyclical terms (0.5=semi-annual, 1=annual)
+    TERMS: list of extra terms
     STDEV: standard deviation of output error
     CONF: confidence interval of output error
     AICc: use second order AIC
@@ -55,6 +56,7 @@
     scipy: Scientific Tools for Python (https://docs.scipy.org/doc/)
 
 UPDATE HISTORY:
+    Updated 04/2023: option to include extra fit terms in the design matrix
     Updated 01/2023: refactored time series analysis functions
     Updated 04/2022: updated docstrings to numpy documentation format
     Updated 05/2021: define int/float precision to prevent deprecation warning
@@ -102,8 +104,9 @@
 import scipy.stats
 import scipy.special
 
-def regress(t_in, d_in, ORDER=1, CYCLES=[0.5,1.0], DATA_ERR=0,
-    WEIGHT=False, RELATIVE=Ellipsis, STDEV=0, CONF=0, AICc=True):
+def regress(t_in, d_in, ORDER=1, CYCLES=[0.5,1.0], TERMS=[],
+    DATA_ERR=0, WEIGHT=False, RELATIVE=Ellipsis, STDEV=0, CONF=0,
+    AICc=True):
     """
     Fits a synthetic signal to data over a time period by
     ordinary or weighted least-squares
@@ -122,6 +125,8 @@ def regress(t_in, d_in, ORDER=1, CYCLES=[0.5,1.0], DATA_ERR=0,
             * ``2``: quadratic
     CYCLES: list, default [0.5, 1.0]
         list of cyclical terms
+    TERMS: list, default []
+        list of extra fit terms
     DATA_ERR: float or list
         Data precision
 
@@ -201,6 +206,7 @@ def regress(t_in, d_in, ORDER=1, CYCLES=[0.5,1.0], DATA_ERR=0,
         t_rel = t_in[RELATIVE].mean()
 
     # create design matrix based on polynomial order and harmonics
+    # with any additional fit terms
     DMAT = []
     # add polynomial orders (0=constant, 1=linear, 2=quadratic)
     for o in range(ORDER+1):
@@ -209,6 +215,9 @@ def regress(t_in, d_in, ORDER=1, CYCLES=[0.5,1.0], DATA_ERR=0,
     for c in CYCLES:
         DMAT.append(np.sin(2.0*np.pi*t_in/np.float64(c)))
         DMAT.append(np.cos(2.0*np.pi*t_in/np.float64(c)))
+    # add additional terms to the design matrix
+    for t in TERMS:
+        DMAT.append(t)
     # take the transpose of the design matrix
     DMAT = np.transpose(DMAT)
 

gravity_toolkit/utilities.py

@@ -1,14 +1,15 @@
 #!/usr/bin/env python
 u"""
 utilities.py
-Written by Tyler Sutterley (03/2023)
+Written by Tyler Sutterley (04/2023)
 Download and management utilities for syncing time and auxiliary files
 
 PYTHON DEPENDENCIES:
     lxml: processing XML and HTML in Python
         https://pypi.python.org/pypi/lxml
 
 UPDATE HISTORY:
+    Updated 04/2023: use release-03 GFZ GravIS SLR and geocenter files
     Updated 03/2023: place boto3 import within try/except statement
     Updated 01/2023: add default ssl context attribute with protocol
     Updated 12/2022: add variables for NASA DAAC and s3 providers
@@ -1867,15 +1868,15 @@ def from_gfz(directory, timeout=None, chunk=8192, verbose=False,
     FILES.append(['isdcftp.gfz-potsdam.de','grace','Level-2','GFZ',
         'RL06_SLR_C20','GFZ_RL06_C20_SLR.dat'])
     FILES.append(['isdcftp.gfz-potsdam.de','grace','GravIS','GFZ',
-        'Level-2B','aux_data','GRAVIS-2B_GFZOP_GRACE+SLR_LOW_DEGREES_0002.dat'])
+        'Level-2B','aux_data','GRAVIS-2B_GFZOP_GRACE+SLR_LOW_DEGREES_0003.dat'])
     # get each file
     for FILE in FILES:
         local = os.path.join(directory,FILE[-1])
         from_ftp(FILE,timeout=timeout,local=local,hash=get_hash(local),
             chunk=chunk,verbose=verbose,fid=fid,mode=mode)
     # GravIS geocenter file
     FILE = ['isdcftp.gfz-potsdam.de','grace','GravIS','GFZ','Level-2B',
-        'aux_data','GRAVIS-2B_GFZOP_GEOCENTER_0002.dat']
+        'aux_data','GRAVIS-2B_GFZOP_GEOCENTER_0003.dat']
     local = os.path.join(directory,'geocenter',FILE[-1])
     from_ftp(FILE,timeout=timeout,local=local,hash=get_hash(local),
         chunk=chunk,verbose=verbose,fid=fid,mode=mode)