From 69e84fe9f169f6b744781a8ba1544274d329cc6d Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Thu, 2 Mar 2023 19:51:02 -0700 Subject: [PATCH] Fix byterange handling of some URLS re: Issue The byterange handling of the following URLS fails. ### Problem 1: "https://crudata.uea.ac.uk/cru/data/temperature/HadCRUT.4.6.0.0.median.nc#mode=bytes" It turns out that byterange in hdf5 has two possible targets: S3 and not-S3 (e.g. a thredds server or the crudata URL above). Each uses a different HDF5 Virtual File Driver (VFD). I incorrectly set up the byterange code in libhdf5 so that it would choose one or the other of the two VFD's for any netcdf-c library build. The fix is to allow it to choose either one at run-time. ### Problem 2: "https://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3" When given what appears to be an S3-related URL, the netcdf-c library code converts it into a canonical, so-called "path" format. In casing out the possible input URL formats, I missed the case where the host contains the bucket ("noaa-goes16"), but not the region. So the fix was to check for this case. ## Misc. Related Changes 1. Since S3 is used in more than just NCZarr, I changed the automake/cmake options to replace "--enable-nczarr-s3" with "--enable-s3", but keeping the former option as a synonym for the latter. This also entailed cleaning up libnetcdf.settings WRT S3 support 2. Added the above URLS as additional test cases ## Misc. Un-Related Changes 1. CURLOPT_PUT is deprecated in favor to CURLOPT_UPLOAD 2. Fix some minor warnings ## Open Problems * Under Ubuntu, either libcrypto or aws-sdk-cpp has a memory leak. --- CMakeLists.txt | 34 ++-- RELEASE_NOTES.md | 3 +- config.h.cmake.in | 16 +- configure.ac | 73 ++++---- include/netcdf_meta.h.in | 2 +- libdispatch/dhttp.c | 4 +- libdispatch/drc.c | 3 +- libdispatch/ds3util.c | 53 ++++-- libdispatch/ncs3sdk.cpp | 16 +- libhdf5/H5FDhttp.h | 2 + libhdf5/hdf5debug.c | 2 +- libhdf5/hdf5open.c | 67 +++---- libncxml/ncxml_xml2.c | 2 +- libnczarr/zdebug.c | 2 +- libnetcdf.settings.in | 3 +- nc_test/Makefile.am | 2 +- nc_test/ref_tst_http_nc4e.cdl | 331 ++++++++++++++++++++++++++++++++++ nc_test/ref_tst_http_nc4f.cdl | 50 +++++ nc_test/test_byterange.sh | 35 ++-- nc_test4/tst_quantize.c | 7 + nczarr_test/CMakeLists.txt | 4 +- nczarr_test/Makefile.am | 2 +- nczarr_test/run_interop.sh | 39 ++-- nczarr_test/run_scalar.sh | 1 - test_common.in | 7 +- 25 files changed, 600 insertions(+), 160 deletions(-) create mode 100644 nc_test/ref_tst_http_nc4e.cdl create mode 100644 nc_test/ref_tst_http_nc4f.cdl diff --git a/CMakeLists.txt b/CMakeLists.txt index 0689927db0..31eaf99bb7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1280,19 +1280,26 @@ IF(NOT WIN32) ENDIF() ENDIF() -# Options to Extend NCZarr support -OPTION(ENABLE_NCZARR_S3 "Enable NCZarr S3 support." OFF) +# Options for S3 Support +OPTION(ENABLE_S3 "Enable S3 support." OFF) +OPTION(ENABLE_NCZARR_S3 "Enable NCZarr S3 support; Deprecated in favor of ENABLE_S3" OFF) OPTION(ENABLE_NCZARR_S3_TESTS "Enable NCZarr S3 tests." OFF) -IF(ENABLE_NCZARR_S3_TESTS AND NOT ENABLE_NCZARR_S3) - message(FATAL_ERROR "NCZarr S3 support is disabled; please specify option -DENABLE_NCZARR_S3_TESTS=no") - SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "NCARR S3 TESTS" FORCE) +# ENABLE_NCZARR_S3 is now an alias for ENABLE_S3 (but...) +if (NOT ENABLE_S3 AND ENABLE_NCZARR_S3) + SET(ENABLE_S3 ON CACHE BOOL "NCARR S3" FORCE) # For back compatibility ENDIF() +UNSET(ENABLE_NCZARR_S3) -# Note we check for the library after checking for enable_nczarr_s3 +IF(ENABLE_NCZARR_S3_TESTS AND NOT ENABLE_S3) + message(FATAL_ERROR "S3 support is disabled; please specify option -DENABLE_NCZARR_S3_TESTS=NO") + SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "NCZARR S3 TESTS" FORCE) +ENDIF() + +# Note we check for the library after checking for enable_s3 # because for some reason this screws up if we unconditionally test for sdk # and it is not available. Fix someday -IF(ENABLE_NCZARR_S3) +IF(ENABLE_S3) # See if aws-s3-sdk is available find_package(AWSSDK REQUIRED COMPONENTS s3;core) IF(AWSSDK_FOUND) @@ -1307,9 +1314,9 @@ ELSE() ENDIF() IF(NOT ENABLE_S3_SDK) - IF(ENABLE_NCZARR_S3 OR ENABLE_NCZARR_S3_TESTS) - message(FATAL_ERROR "S3 support library not found; please specify option -DENABLE_NCZARR_S3=NO") - SET(ENABLE_NCZARR_S3 OFF CACHE BOOL "NCZARR S3 support" FORCE) + IF(ENABLE_S3) + message(FATAL_ERROR "S3 support library not found; please specify option -DENABLE_S3=NO") + SET(ENABLE_S3 OFF CACHE BOOL "S3 support" FORCE) SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "S3 tests" FORCE) ENDIF() ENDIF() @@ -1335,8 +1342,8 @@ IF(NOT FOUND_CURL) SET(ENABLE_HDF5_ROS3 OFF CACHE BOOL "Use ROS3" FORCE) ENDIF() - IF(ENABLE_NCZARR_S3) - MESSAGE(FATAL_ERROR "NCZarr S3 support specified, CURL libraries are not found.") + IF(ENABLE_S3) + MESSAGE(FATAL_ERROR "S3 support specified, but CURL libraries are not found.") ENDIF() ENDIF(NOT FOUND_CURL) @@ -2544,7 +2551,8 @@ is_enabled(ENABLE_ZERO_LENGTH_COORD_BOUND RELAX_COORD_BOUND) is_enabled(USE_CDF5 HAS_CDF5) is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL) is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS) -is_enabled(ENABLE_NCZARR_S3 HAS_NCZARR_S3) +is_enabled(ENABLE_S3 HAS_S3) +is_enabled(ENABLE_S3_SDK HAS_S3_SDK) is_enabled(ENABLE_NCZARR HAS_NCZARR) is_enabled(ENABLE_NCZARR_S3_TESTS DO_NCZARR_S3_TESTS) is_enabled(ENABLE_NCZARR_ZIP HAS_NCZARR_ZIP) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 511e4f9138..69fdeed20d 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,8 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.2 - TBD -* Fix 'make distcheck' error in run_interop.sh. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). +* Fix byterange failures for certain URLs. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). +* Fix 'make distcheck' error in run_interop.sh. See [Github #2631](https://github.com/Unidata/netcdf-c/pull/2631). * Update `nc-config` to remove inclusion from automatically-detected `nf-config` and `ncxx-config` files, as the wrong files could be included in the output. This is in support of [GitHub #2274](https://github.com/Unidata/netcdf-c/issues/2274). * Update H5FDhttp.[ch] to work with HDF5 version 1.14.0. See [Github #2615](https://github.com/Unidata/netcdf-c/pull/2615). diff --git a/config.h.cmake.in b/config.h.cmake.in index 7273971f31..80dc01ee77 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -136,23 +136,23 @@ are set when opening a binary file on Windows. */ /* if true, do remote tests */ #cmakedefine ENABLE_DAP_REMOTE_TESTS 1 +/* if true, enable S3 support */ +#cmakedefine ENABLE_S3 1 + +/* if true, S3 SDK is available */ +#cmakedefine ENABLE_S3_SDK 1 + /* if true, enable NCZARR */ #cmakedefine ENABLE_NCZARR 1 /* if true, enable nczarr filter support */ #cmakedefine ENABLE_NCZARR_FILTERS 1 -/* if true, enable S3 support */ -#cmakedefine ENABLE_NCZARR_S3 1 - -/* if true, enable nczarr zip support */ -#cmakedefine ENABLE_NCZARR_ZIP 1 - /* if true, enable S3 testing*/ #cmakedefine ENABLE_NCZARR_S3_TESTS 1 -/* if true, S3 SDK is available */ -#cmakedefine ENABLE_S3_SDK 1 +/* if true, enable nczarr zip support */ +#cmakedefine ENABLE_NCZARR_ZIP 1 /* if true, Allow dynamically loaded plugins */ #cmakedefine ENABLE_PLUGINS 1 diff --git a/configure.ac b/configure.ac index 94bf84b4d7..b6f3bd3405 100644 --- a/configure.ac +++ b/configure.ac @@ -565,7 +565,7 @@ if test "x$enable_libxml2" = xyes; then if test -z "$NC_XML2_CONFIG"; then AC_MSG_ERROR([Cannot find xml2-config utility. Either install the libxml2 development package, or re-run configure with --disable-libxml2 to use the bundled xml2 parser]) fi - # We can optionally use libxml2 for DAP4, if available + # We can optionally use libxml2 for DAP4 AC_CHECK_LIB([xml2],[xmlReadMemory],[have_libxml2=yes],[have_libxml2=no]) if test "x$have_libxml2" = "xyes" ; then AC_SEARCH_LIBS([xmlReadMemory],[xml2 xml2.dll cygxml2.dll], [],[]) @@ -782,58 +782,59 @@ AC_MSG_RESULT([${have_sz}]) AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll],[have_zip=yes],[have_zip=no]) AC_MSG_CHECKING([whether libzip library is available]) AC_MSG_RESULT([${have_zip}]) - enable_nczarr_zip=${have_zip} # alias if test "x$enable_nczarr" = xno ; then enable_nczarr_zip=no fi - - AC_MSG_CHECKING([whether nczarr zip support is enabled]) AC_MSG_RESULT([${enable_nczarr_zip}]) if test "x$enable_nczarr_zip" = xyes ; then AC_DEFINE([ENABLE_NCZARR_ZIP], [1], [If true, then libzip found]) fi -AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes]) # Check for enabling of S3 support -AC_MSG_CHECKING([whether netcdf zarr S3 support should be enabled]) +AC_MSG_CHECKING([whether netcdf S3 support should be enabled]) +AC_ARG_ENABLE([s3], + [AS_HELP_STRING([--enable-s3], + [enable netcdf S3 support])]) +test "x$enable_s3" = xyes || enable_s3=no +AC_MSG_RESULT($enable_s3) + +# --enable-nczarr-s3 is synonym for --enable-s3 (but...) +AC_MSG_CHECKING([whether netcdf NCZarr S3 support should be enabled]) AC_ARG_ENABLE([nczarr-s3], [AS_HELP_STRING([--enable-nczarr-s3], - [enable netcdf zarr S3 support; make sure to set LDFLAGS])]) -test "x$enable_nczarr_s3" = xyes || enable_nczarr_s3=no -if test "x$enable_nczarr" = xno ; then -enable_nczarr_s3=no + [enable netcdf NCZarr S3 support; Deprecated in favor of --enable-s3])]) +# Set enable_s3 instead of enable_nczarr_s3 +if test "x$enable_s3" = xno && test "x$enable_nczarr_s3" = xyes ; then +enable_s3=yes # back compatibility fi -AC_MSG_RESULT($enable_nczarr_s3) +unset enable_nczarr_s3 +AC_MSG_RESULT($enable_s3) -# Note we check for the library after checking for enable_nczarr_s3 -# because for some reason this screws up if we unconditionally test for sdk +# Note we check for the library after checking for enable_s3 +# because for some reason this fails if we unconditionally test for sdk # and it is not available. Fix someday -enable_s3_sdk=no -if test "x$enable_nczarr_s3" = xyes ; then +if test "x$enable_s3" = xyes ; then # See if we have the s3 aws library # Check for the AWS S3 SDK library AC_LANG_PUSH([C++]) AC_SEARCH_LIBS([aws_allocator_is_valid],[aws-c-common aws-cpp-sdk-s3 aws-cpp-sdk-core], [enable_s3_sdk=yes],[enable_s3_sdk=no]) AC_LANG_POP +else +enable_s3_sdk=no fi AC_MSG_CHECKING([whether AWS S3 SDK library is available]) AC_MSG_RESULT([$enable_s3_sdk]) if test "x$enable_s3_sdk" = xno ; then -AC_MSG_WARN([No S3 library available; disabling S3 support]) -enable_nczarr_s3=no -fi - -if test "x$enable_s3_sdk" = xyes ; then -AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found]) +AC_MSG_WARN([No S3 library available => S3 support disabled]) +enable_S3=no fi -AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$enable_s3_sdk" = xyes]) # Check for enabling S3 testing AC_MSG_CHECKING([whether netcdf zarr S3 testing should be enabled]) @@ -844,20 +845,22 @@ test "x$enable_nczarr_s3_tests" = xyes || enable_nczarr_s3_tests=no AC_MSG_RESULT($enable_nczarr_s3_tests) # Disable S3 tests if S3 support is disabled -if test "x$enable_nczarr_s3" = xno && test "x$enable_nczarr_s3_tests" = xyes ; then - AC_MSG_ERROR([NCZarr S3 support is disabled; please remove option --enable-nczarr-s3-tests]) +if test "x$enable_nczarr_s3_tests" = xyes ; then + if test "x$enable_s3" = xno ; then + AC_MSG_ERROR([S3 support is disabled => no testing]) enable_nczarr_s3_tests=no + fi fi -if test "x$enable_nczarr_s3" = xyes ; then - AC_DEFINE([ENABLE_NCZARR_S3], [1], [if true, build libnczarr with S3 support enabled]) +if test "x$enable_s3" = xyes ; then + AC_DEFINE([ENABLE_S3], [1], [if true, build netcdf-c with S3 support enabled]) fi -AM_CONDITIONAL(ENABLE_NCZARR_S3, [test "x$enable_nczarr_s3" = xyes]) - if test "x$enable_nczarr_s3_tests" = xyes ; then AC_DEFINE([ENABLE_NCZARR_S3_TESTS], [1], [if true, build libnczarr with S3 tests enabled]) fi -AM_CONDITIONAL(ENABLE_NCZARR_S3_TESTS, [test "x$enable_nczarr_s3_tests" = xyes]) +if test "x$enable_s3_sdk" = xyes ; then +AC_DEFINE([ENABLE_S3_SDK], [1], [If true, then S3 sdk was found]) +fi if test "x$enable_nczarr_s3_tests" = xyes ; then AC_MSG_WARN([*** DO NOT ENABLE_NCZARR_S3_TESTS UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***]) @@ -1191,7 +1194,7 @@ AC_CHECK_HEADERS([sys/param.h]) AC_CHECK_HEADERS([libgen.h]) #AC_CHECK_HEADERS([locale.h]) AC_HEADER_STDC -AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h time.h dirent.h stdint.h]) +AC_CHECK_HEADERS([locale.h stdio.h stdarg.h fcntl.h malloc.h stdlib.h string.h strings.h unistd.h sys/stat.h getopt.h sys/time.h sys/types.h time.h dirent.h stdint.h ctype.h]) # Do sys/resource.h separately #AC_CHECK_HEADERS([sys/resource.h],[havesysresource=1],[havesysresource=0]) @@ -1284,6 +1287,7 @@ AC_ARG_ENABLE([byterange], [allow byte-range I/O])]) test "x$enable_byterange" = xno || enable_byterange=yes AC_MSG_RESULT($enable_byterange) + # Need curl for byte ranges if test "x$found_curl" = xno && test "x$enable_byterange" = xyes ; then AC_MSG_ERROR([curl required for byte range support. Install curl or build without --enable-byterange.]) @@ -1822,8 +1826,12 @@ AM_CONDITIONAL(ENABLE_METADATA_PERF, [test x$enable_metadata_perf = xyes]) AM_CONDITIONAL(ENABLE_BYTERANGE, [test "x$enable_byterange" = xyes]) AM_CONDITIONAL(RELAX_COORD_BOUND, [test "xyes" = xyes]) AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ]) -AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes]) +# We need to simplify the set of S3 and Zarr flag combinations +AM_CONDITIONAL(ENABLE_S3, [test "x$enable_s3" = xyes]) AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$enable_s3_sdk" = xyes]) +AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes]) +AM_CONDITIONAL(ENABLE_NCZARR_S3_TESTS, [test "x$enable_nczarr_s3_tests" = xyes]) +AM_CONDITIONAL(ENABLE_NCZARR_ZIP, [test "x$enable_nczarr_zip" = xyes]) AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes]) AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes]) AM_CONDITIONAL(HAVE_H5Z_SZIP, [test "x$enable_hdf5_szip" = xyes]) @@ -1927,7 +1935,7 @@ AC_SUBST(HAS_ERANGE_FILL,[$enable_erange_fill]) AC_SUBST(HAS_BYTERANGE,[$enable_byterange]) AC_SUBST(RELAX_COORD_BOUND,[yes]) AC_SUBST([HAS_PAR_FILTERS], [$hdf5_supports_par_filters]) -AC_SUBST(HAS_NCZARR_S3,[$enable_nczarr_s3]) +AC_SUBST(HAS_S3,[$enable_s3]) AC_SUBST(HAS_NCZARR,[$enable_nczarr]) AC_SUBST(DO_NCZARR_S3_TESTS,[$enable_nczarr_s3_tests]) AC_SUBST(HAS_NCZARR_ZIP,[$enable_nczarr_zip]) @@ -2057,6 +2065,7 @@ AX_SET_META([NC_HAS_ERANGE_FILL], [$enable_erange_fill],[yes]) AX_SET_META([NC_HAS_PAR_FILTERS], [$hdf5_supports_par_filters],[yes]) AX_SET_META([NC_HAS_BYTERANGE],[$enable_byterange],[yes]) AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes]) +AX_SET_META([NC_HAS_S3],[$enable_s3],[yes]) AX_SET_META([NC_HAS_MULTIFILTERS],[$has_multifilters],[yes]) AX_SET_META([NC_HAS_LOGGING],[$enable_logging],[yes]) AX_SET_META([NC_HAS_QUANTIZE],[$enable_quantize],[yes]) diff --git a/include/netcdf_meta.h.in b/include/netcdf_meta.h.in index 705cb6a526..c4e613d218 100644 --- a/include/netcdf_meta.h.in +++ b/include/netcdf_meta.h.in @@ -66,7 +66,7 @@ #define NC_HAS_QUANTIZE @NC_HAS_QUANTIZE@ /*!< Quantization support. */ #define NC_HAS_ZSTD @NC_HAS_ZSTD@ /*!< Zstd support. */ #define NC_HAS_BENCHMARKS @NC_HAS_BENCHMARKS@ /*!< Benchmarks. */ - +#define NC_HAS_S3 @NC_HAS_S3@ /*!< Amazon S3 Support. */ #define NC_HAS_BLOSC @NC_HAS_BLOSC@ /*!< Blosc Support. */ #define NC_HAS_BZ2 @NC_HAS_BZ2@ /*!< bzip2 support */ diff --git a/libdispatch/dhttp.c b/libdispatch/dhttp.c index 3b8eba6e2e..c97eddde8c 100644 --- a/libdispatch/dhttp.c +++ b/libdispatch/dhttp.c @@ -135,7 +135,7 @@ nc_http_reset(NC_HTTP_STATE* state) if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;} cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_NOBODY, 0L)); if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;} - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_PUT, 0L)); + cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_UPLOAD, 0L)); if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;} cstat = curl_easy_setopt(state->curl, CURLOPT_CUSTOMREQUEST, NULL); if(cstat != CURLE_OK) {stat = NC_ECURL; goto done;} @@ -168,7 +168,7 @@ nc_http_set_method(NC_HTTP_STATE* state, HTTPMETHOD method) cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_NOBODY, 1L)); break; case HTTPPUT: - cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_PUT, 1L)); + cstat = CURLERR(curl_easy_setopt(state->curl, CURLOPT_UPLOAD, 1L)); break; case HTTPDELETE: cstat = curl_easy_setopt(state->curl, CURLOPT_CUSTOMREQUEST, "DELETE"); diff --git a/libdispatch/drc.c b/libdispatch/drc.c index 160dff23de..07f7ed95ee 100644 --- a/libdispatch/drc.c +++ b/libdispatch/drc.c @@ -484,7 +484,8 @@ rccompile(const char* filepath) nclog(NCLOGERR, "Malformed [url] in %s entry: %s",filepath,line); continue; } - { NCURI* newuri = NULL; + if(NC_iss3(uri)) { + NCURI* newuri = NULL; /* Rebuild the url to S3 "path" format */ nullfree(bucket); if((ret = NC_s3urlrebuild(uri,&newuri,&bucket,NULL))) goto done; diff --git a/libdispatch/ds3util.c b/libdispatch/ds3util.c index 3d6ba67244..21ab90de6c 100644 --- a/libdispatch/ds3util.c +++ b/libdispatch/ds3util.c @@ -76,41 +76,48 @@ NC_s3urlrebuild(NCURI* url, NCURI** newurlp, char** bucketp, char** outregionp) if((stat = NC_split_delim(url->path,'/',pathsegments))) goto done; /* Distinguish path-style from virtual-host style from s3: and from other. - Virtual: https://bucket-name.s3.Region.amazonaws.com/ - Path: https://s3.Region.amazonaws.com/bucket-name/ - S3: s3://bucket-name/ - Other: https:///bucketname/ + Virtual: https://bucket-name.s3.Region.amazonaws.com/ (1) + or: https://bucket-name.s3.amazonaws.com/ -- region defaults to us-east-1 (2) + Path: https://s3.Region.amazonaws.com/bucket-name/ (3) + or: https://s3.amazonaws.com/bucket-name/ -- region defaults to us-east-1 (4) + S3: s3://bucket-name/ (5) + Other: https:///bucketname/ (6) */ if(url->host == NULL || strlen(url->host) == 0) {stat = NC_EURL; goto done;} - if(strcmp(url->protocol,"s3")==0 && nclistlength(hostsegments)==1) { - bucket = strdup(url->host); - region = NULL; /* unknown at this point */ + if(strcmp(url->protocol,"s3")==0 && nclistlength(hostsegments)==1) { /* Case (5) */ + bucket = nclistremove(hostsegments,0); + /* region unknown at this point */ } else if(endswith(url->host,AWSHOST)) { /* Virtual or path */ + /* If we find a bucket as part of the host, then remove it */ switch (nclistlength(hostsegments)) { default: stat = NC_EURL; goto done; - case 4: - if(strcasecmp(nclistget(hostsegments,0),"s3")!=0) + case 3: /* Case (4) */ + /* region unknown at this point */ + /* bucket unknown at this point */ + break; + case 4: /* Case (2) or (3) */ + if(strcasecmp(nclistget(hostsegments,1),"s3")==0) { /* Case (2) */ + /* region unknown at this point */ + bucket = nclistremove(hostsegments,0); /* Note removal */ + } else if(strcasecmp(nclistget(hostsegments,0),"s3")==0) { /* Case (3) */ + region = strdup(nclistget(hostsegments,1)); + /* bucket unknown at this point */ + } else /* ! (2) and !(3) => error */ {stat = NC_EURL; goto done;} - region = strdup(nclistget(hostsegments,1)); - if(nclistlength(pathsegments) > 0) - bucket = nclistremove(pathsegments,0); break; - case 5: + case 5: /* Case (1) */ if(strcasecmp(nclistget(hostsegments,1),"s3")!=0) {stat = NC_EURL; goto done;} region = strdup(nclistget(hostsegments,2)); - bucket = strdup(nclistget(hostsegments,0)); + bucket = strdup(nclistremove(hostsegments,0)); break; } - } else { + } else { /* Presume Case (6) */ if((host = strdup(url->host))==NULL) {stat = NC_ENOMEM; goto done;} /* region is unknown */ - region = NULL; - /* bucket is assumed to be start of the path */ - if(nclistlength(pathsegments) > 0) - bucket = nclistremove(pathsegments,0); + /* bucket is unknown */ } /* If region is null, use default */ if(region == NULL) { @@ -119,6 +126,14 @@ NC_s3urlrebuild(NCURI* url, NCURI** newurlp, char** bucketp, char** outregionp) if((stat = NC_getdefaults3region(url,®ion0))) goto done; region = strdup(region0); } + /* if bucket is null, use first segment of the path, if any */ + if(bucket == NULL) { + if(nclistlength(pathsegments) > 0) + bucket = nclistremove(pathsegments,0); + } + assert(bucket != NULL); + /* bucket may still be null */ + if(host == NULL) { /* Construct the revised host */ ncbytescat(buf,"s3."); ncbytescat(buf,region); diff --git a/libdispatch/ncs3sdk.cpp b/libdispatch/ncs3sdk.cpp index 1b0feaec02..d415052d2f 100644 --- a/libdispatch/ncs3sdk.cpp +++ b/libdispatch/ncs3sdk.cpp @@ -635,16 +635,20 @@ s3commonprefixes(Aws::Vector list, char*** keysp) if(!stat) { i = 0; for (auto const &s3prefix : list) { - char* p; char* p1; - size_t len; + char* p; const char* px; char* p1; + size_t len, alloc; const Aws::String& prefix = s3prefix.GetPrefix(); len = prefix.length(); - if((p = (char*) malloc(len+1+1))==NULL) /* for nul + leading '/' */ + alloc = len + 1 + 1; /* for nul + leading '/' */ + if((p = (char*) malloc(alloc))==NULL) stat = NC_ENOMEM; if(stat == NC_NOERR) { - if(*p == '/') {p1 = p;} else {*p = '/'; p1 = p+1;} - memcpy(p1,prefix.c_str(),len); - p1[len] = '\0'; + px = prefix.c_str(); + p1 = p; + if(*px != '/') *p1++ = '/'; + memcpy(p1,px,len); + p1 += len; + *p1 = '\0'; keys[i++] = p; } } diff --git a/libhdf5/H5FDhttp.h b/libhdf5/H5FDhttp.h index 2e0e11db82..7794d57376 100644 --- a/libhdf5/H5FDhttp.h +++ b/libhdf5/H5FDhttp.h @@ -27,6 +27,8 @@ #ifndef H5FDHTTP_H #define H5FDHTTP_H +#define S3_REGION_DEFAULT "us-east-1" + #include "H5Ipublic.h" #if H5_VERSION_GE(1,14,0) diff --git a/libhdf5/hdf5debug.c b/libhdf5/hdf5debug.c index ca5c6cd68e..5462eb5475 100644 --- a/libhdf5/hdf5debug.c +++ b/libhdf5/hdf5debug.c @@ -44,7 +44,7 @@ nch5breakpoint(int err) # endif #endif #endif - return err; + return ncbreakpoint(err); } int diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 52f3653fee..cb2491ff9f 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -871,12 +871,18 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) char* awsregion0 = NULL; #ifdef ENABLE_HDF5_ROS3 H5FD_ros3_fapl_t fa; - char* hostport = NULL; const char* profile0 = NULL; const char* awsaccessid0 = NULL; const char* awssecretkey0 = NULL; + int iss3 = NC_iss3(h5->uri); - if(NC_iss3(h5->uri)) { + fa.version = H5FD_CURR_ROS3_FAPL_T_VERSION; + fa.authenticate = (hbool_t)0; + fa.aws_region[0] = '\0'; + fa.secret_id[0] = '\0'; + fa.secret_key[0] = '\0'; + + if(iss3) { /* Rebuild the URL */ NCURI* newuri = NULL; if((retval = NC_s3urlrebuild(h5->uri,&newuri,NULL,&awsregion0))) goto exit; @@ -884,38 +890,35 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) {retval = NC_EURL; goto exit;} ncurifree(h5->uri); h5->uri = newuri; - } - hostport = NC_combinehostport(h5->uri); - if((retval = NC_getactives3profile(h5->uri,&profile0))) - BAIL(retval); - - fa.version = 1; - fa.aws_region[0] = '\0'; - fa.secret_id[0] = '\0'; - fa.secret_key[0] = '\0'; - if((retval = NC_s3profilelookup(profile0,AWS_ACCESS_KEY_ID,&awsaccessid0))) - BAIL(retval); - if((retval = NC_s3profilelookup(profile0,AWS_SECRET_ACCESS_KEY,&awssecretkey0))) - BAIL(retval); - if(awsaccessid0 == NULL || awssecretkey0 == NULL) { - /* default, non-authenticating, "anonymous" fapl configuration */ - fa.authenticate = (hbool_t)0; - } else { - fa.authenticate = (hbool_t)1; - if(awsregion0) + if((retval = NC_getactives3profile(h5->uri,&profile0))) + BAIL(retval); + if((retval = NC_s3profilelookup(profile0,AWS_ACCESS_KEY_ID,&awsaccessid0))) + BAIL(retval); + if((retval = NC_s3profilelookup(profile0,AWS_SECRET_ACCESS_KEY,&awssecretkey0))) + BAIL(retval); + if(awsregion0 == NULL) + awsregion0 = strdup(S3_REGION_DEFAULT); + if(awsaccessid0 == NULL || awssecretkey0 == NULL ) { + /* default, non-authenticating, "anonymous" fapl configuration */ + fa.authenticate = (hbool_t)0; + } else { + fa.authenticate = (hbool_t)1; + assert(awsregion0 != NULL && strlen(awsregion0) > 0); + assert(awsaccessid0 != NULL && strlen(awsaccessid0) > 0); + assert(awssecretkey0 != NULL && strlen(awssecretkey0) > 0); strlcat(fa.aws_region,awsregion0,H5FD_ROS3_MAX_REGION_LEN); - strlcat(fa.secret_id, awsaccessid0, H5FD_ROS3_MAX_SECRET_ID_LEN); - strlcat(fa.secret_key, awssecretkey0, H5FD_ROS3_MAX_SECRET_KEY_LEN); - } - nullfree(hostport); - /* create and set fapl entry */ - if(H5Pset_fapl_ros3(fapl_id, &fa) < 0) - BAIL(NC_EHDFERR); -#else - /* Configure FAPL to use our byte-range file driver */ - if (H5Pset_fapl_http(fapl_id) < 0) - BAIL(NC_EHDFERR); + strlcat(fa.secret_id, awsaccessid0, H5FD_ROS3_MAX_SECRET_ID_LEN); + strlcat(fa.secret_key, awssecretkey0, H5FD_ROS3_MAX_SECRET_KEY_LEN); + } + /* create and set fapl entry */ + if(H5Pset_fapl_ros3(fapl_id, &fa) < 0) + BAIL(NC_EHDFERR); + } else #endif /*ENABLE_ROS3*/ + {/* Configure FAPL to use our byte-range file driver */ + if (H5Pset_fapl_http(fapl_id) < 0) + BAIL(NC_EHDFERR); + } /* Open the HDF5 file. */ if ((h5->hdfid = nc4_H5Fopen((newpath?newpath:path), flags, fapl_id)) < 0) BAIL(NC_EHDFERR); diff --git a/libncxml/ncxml_xml2.c b/libncxml/ncxml_xml2.c index cfe49a3f3e..c32b1cc857 100644 --- a/libncxml/ncxml_xml2.c +++ b/libncxml/ncxml_xml2.c @@ -51,7 +51,7 @@ const char* ncxml_name(ncxml_t xml0) { xmlNode* xml = (xmlNode*)xml0; - return (xml?xml->name:(const char*)NULL); + return (const char*)(xml?xml->name:NULL); } char* diff --git a/libnczarr/zdebug.c b/libnczarr/zdebug.c index 26b2180d6e..2a1ad4efaf 100644 --- a/libnczarr/zdebug.c +++ b/libnczarr/zdebug.c @@ -19,7 +19,7 @@ struct ZUTEST* zutest = NULL; int zbreakpoint(int err) { - return err; + return ncbreakpoint(err); } int diff --git a/libnetcdf.settings.in b/libnetcdf.settings.in index ea31a830d4..d467c7b0fb 100644 --- a/libnetcdf.settings.in +++ b/libnetcdf.settings.in @@ -40,8 +40,9 @@ DAP2 Support: @HAS_DAP2@ DAP4 Support: @HAS_DAP4@ Byte-Range Support: @HAS_BYTERANGE@ +S3 Support: @HAS_S3@ + NCZarr Support: @HAS_NCZARR@ -NCZarr S3 Support: @HAS_NCZARR_S3@ NCZarr Zip Support: @HAS_NCZARR_ZIP@ Diskless Support: @HAS_DISKLESS@ diff --git a/nc_test/Makefile.am b/nc_test/Makefile.am index e7e7e5dfc0..73424066c6 100644 --- a/nc_test/Makefile.am +++ b/nc_test/Makefile.am @@ -116,7 +116,7 @@ ref_tst_diskless3_create.cdl ref_tst_diskless3_open.cdl \ run_inmemory.sh run_mmap.sh f03tst_open_mem.nc test_byterange.sh \ ref_tst_http_nc3.cdl ref_tst_http_nc4a.cdl ref_tst_http_nc4b.cdl \ ref_tst_http_nc4c.cdl CMakeLists.txt run_pnetcdf_tests.sh.in \ -tst_misc.c tst_norm.c +tst_misc.c tst_norm.c ref_tst_http_nc4e.cdl ref_tst_http_nc4f.cdl # These files are created by the tests. CLEANFILES = nc_test_*.nc tst_*.nc t_nc.nc large_files.nc \ diff --git a/nc_test/ref_tst_http_nc4e.cdl b/nc_test/ref_tst_http_nc4e.cdl new file mode 100644 index 0000000000..028d6a8a8b --- /dev/null +++ b/nc_test/ref_tst_http_nc4e.cdl @@ -0,0 +1,331 @@ +netcdf OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562 { +dimensions: + y = 10848 ; + x = 10848 ; + number_of_time_bounds = 2 ; + band = 1 ; + number_of_image_bounds = 2 ; + num_star_looks = 24 ; +variables: + short Rad(y, x) ; + Rad:_FillValue = 1023s ; + Rad:long_name = "ABI L1b Radiances" ; + Rad:standard_name = "toa_outgoing_radiance_per_unit_wavelength" ; + Rad:_Unsigned = "true" ; + Rad:sensor_band_bit_depth = 10b ; + Rad:valid_range = 0s, 1022s ; + Rad:scale_factor = 0.8121064f ; + Rad:add_offset = -25.93665f ; + Rad:units = "W m-2 sr-1 um-1" ; + Rad:resolution = "y: 0.000028 rad x: 0.000028 rad" ; + Rad:coordinates = "band_id band_wavelength t y x" ; + Rad:grid_mapping = "goes_imager_projection" ; + Rad:cell_methods = "t: point area: point" ; + Rad:ancillary_variables = "DQF" ; + byte DQF(y, x) ; + DQF:_FillValue = -1b ; + DQF:long_name = "ABI L1b Radiances data quality flags" ; + DQF:standard_name = "status_flag" ; + DQF:_Unsigned = "true" ; + DQF:valid_range = 0b, 4b ; + DQF:units = "1" ; + DQF:coordinates = "band_id band_wavelength t y x" ; + DQF:grid_mapping = "goes_imager_projection" ; + DQF:cell_methods = "t: point area: point" ; + DQF:flag_values = 0b, 1b, 2b, 3b, 4b ; + DQF:flag_meanings = "good_pixel_qf conditionally_usable_pixel_qf out_of_range_pixel_qf no_value_pixel_qf focal_plane_temperature_threshold_exceeded_qf" ; + DQF:number_of_qf_values = 5b ; + DQF:percent_good_pixel_qf = 0.998999f ; + DQF:percent_conditionally_usable_pixel_qf = 5.e-07f ; + DQF:percent_out_of_range_pixel_qf = 0.0009925f ; + DQF:percent_no_value_pixel_qf = 8.e-06f ; + DQF:percent_focal_plane_temperature_threshold_exceeded_qf = 0.f ; + double t ; + t:long_name = "J2000 epoch mid-point between the start and end image scan in seconds" ; + t:standard_name = "time" ; + t:units = "seconds since 2000-01-01 12:00:00" ; + t:axis = "T" ; + t:bounds = "time_bounds" ; + short y(y) ; + y:scale_factor = -2.8e-05f ; + y:add_offset = 0.151858f ; + y:units = "rad" ; + y:axis = "Y" ; + y:long_name = "GOES fixed grid projection y-coordinate" ; + y:standard_name = "projection_y_coordinate" ; + short x(x) ; + x:scale_factor = 2.8e-05f ; + x:add_offset = -0.151858f ; + x:units = "rad" ; + x:axis = "X" ; + x:long_name = "GOES fixed grid projection x-coordinate" ; + x:standard_name = "projection_x_coordinate" ; + double time_bounds(number_of_time_bounds) ; + time_bounds:long_name = "Scan start and end times in seconds since epoch (2000-01-01 12:00:00)" ; + int goes_imager_projection ; + goes_imager_projection:long_name = "GOES-R ABI fixed grid projection" ; + goes_imager_projection:grid_mapping_name = "geostationary" ; + goes_imager_projection:perspective_point_height = 35786023. ; + goes_imager_projection:semi_major_axis = 6378137. ; + goes_imager_projection:semi_minor_axis = 6356752.31414 ; + goes_imager_projection:inverse_flattening = 298.2572221 ; + goes_imager_projection:latitude_of_projection_origin = 0. ; + goes_imager_projection:longitude_of_projection_origin = -75. ; + goes_imager_projection:sweep_angle_axis = "x" ; + float y_image ; + y_image:long_name = "GOES-R fixed grid projection y-coordinate center of image" ; + y_image:standard_name = "projection_y_coordinate" ; + y_image:units = "rad" ; + y_image:axis = "Y" ; + float y_image_bounds(number_of_image_bounds) ; + y_image_bounds:long_name = "GOES-R fixed grid projection y-coordinate north/south extent of image" ; + y_image_bounds:units = "rad" ; + float x_image ; + x_image:long_name = "GOES-R fixed grid projection x-coordinate center of image" ; + x_image:standard_name = "projection_x_coordinate" ; + x_image:units = "rad" ; + x_image:axis = "X" ; + float x_image_bounds(number_of_image_bounds) ; + x_image_bounds:long_name = "GOES-R fixed grid projection x-coordinate west/east extent of image" ; + x_image_bounds:units = "rad" ; + float nominal_satellite_subpoint_lat ; + nominal_satellite_subpoint_lat:long_name = "nominal satellite subpoint latitude (platform latitude)" ; + nominal_satellite_subpoint_lat:standard_name = "latitude" ; + nominal_satellite_subpoint_lat:_FillValue = -999.f ; + nominal_satellite_subpoint_lat:units = "degrees_north" ; + float nominal_satellite_subpoint_lon ; + nominal_satellite_subpoint_lon:long_name = "nominal satellite subpoint longitude (platform longitude)" ; + nominal_satellite_subpoint_lon:standard_name = "longitude" ; + nominal_satellite_subpoint_lon:_FillValue = -999.f ; + nominal_satellite_subpoint_lon:units = "degrees_east" ; + float nominal_satellite_height ; + nominal_satellite_height:long_name = "nominal satellite height above GRS 80 ellipsoid (platform altitude)" ; + nominal_satellite_height:standard_name = "height_above_reference_ellipsoid" ; + nominal_satellite_height:_FillValue = -999.f ; + nominal_satellite_height:units = "km" ; + float geospatial_lat_lon_extent ; + geospatial_lat_lon_extent:long_name = "geospatial latitude and longitude references" ; + geospatial_lat_lon_extent:geospatial_westbound_longitude = -156.2995f ; + geospatial_lat_lon_extent:geospatial_northbound_latitude = 81.3282f ; + geospatial_lat_lon_extent:geospatial_eastbound_longitude = 6.2995f ; + geospatial_lat_lon_extent:geospatial_southbound_latitude = -81.3282f ; + geospatial_lat_lon_extent:geospatial_lat_center = 0.f ; + geospatial_lat_lon_extent:geospatial_lon_center = -75.f ; + geospatial_lat_lon_extent:geospatial_lat_nadir = 0.f ; + geospatial_lat_lon_extent:geospatial_lon_nadir = -75.f ; + geospatial_lat_lon_extent:geospatial_lat_units = "degrees_north" ; + geospatial_lat_lon_extent:geospatial_lon_units = "degrees_east" ; + byte yaw_flip_flag ; + yaw_flip_flag:long_name = "Flag indicating the spacecraft is operating in yaw flip configuration" ; + yaw_flip_flag:_Unsigned = "true" ; + yaw_flip_flag:_FillValue = -1b ; + yaw_flip_flag:valid_range = 0b, 1b ; + yaw_flip_flag:units = "1" ; + yaw_flip_flag:coordinates = "t" ; + yaw_flip_flag:flag_values = 0b, 1b ; + yaw_flip_flag:flag_meanings = "false true" ; + byte band_id(band) ; + band_id:long_name = "ABI band number" ; + band_id:standard_name = "sensor_band_identifier" ; + band_id:units = "1" ; + float band_wavelength(band) ; + band_wavelength:long_name = "ABI band central wavelength" ; + band_wavelength:standard_name = "sensor_band_central_radiation_wavelength" ; + band_wavelength:units = "um" ; + float esun ; + esun:long_name = "bandpass-weighted solar irradiance at the mean Earth-Sun distance" ; + esun:standard_name = "toa_shortwave_irradiance_per_unit_wavelength" ; + esun:_FillValue = -999.f ; + esun:units = "W m-2 um-1" ; + esun:coordinates = "band_id band_wavelength t" ; + esun:cell_methods = "t: mean" ; + float kappa0 ; + kappa0:long_name = "Inverse of the incoming top of atmosphere radiance at current earth-sun distance (PI d2 esun-1)-1, where d is the ratio of instantaneous Earth-Sun distance divided by the mean Earth-Sun distance, esun is the bandpass-weighted solar irradiance and PI is a standard constant used to convert ABI L1b radiance to reflectance" ; + kappa0:_FillValue = -999.f ; + kappa0:units = "(W m-2 um-1)-1" ; + kappa0:coordinates = "band_id band_wavelength t" ; + kappa0:cell_methods = "t: mean" ; + float planck_fk1 ; + planck_fk1:long_name = "wavenumber-dependent coefficient (2 h c2/ nu3) used in the ABI emissive band monochromatic brightness temperature computation, where nu =central wavenumber and h and c are standard constants" ; + planck_fk1:_FillValue = -999.f ; + planck_fk1:units = "W m-1" ; + planck_fk1:coordinates = "band_id band_wavelength" ; + float planck_fk2 ; + planck_fk2:long_name = "wavenumber-dependent coefficient (h c nu/b) used in the ABI emissive band monochromatic brightness temperature computation, where nu = central wavenumber and h, c, and b are standard constants" ; + planck_fk2:_FillValue = -999.f ; + planck_fk2:units = "K" ; + planck_fk2:coordinates = "band_id band_wavelength" ; + float planck_bc1 ; + planck_bc1:long_name = "spectral bandpass correction offset for brightness temperature (B(nu) - bc_1)/bc_2 where B()=planck_function() and nu=wavenumber" ; + planck_bc1:_FillValue = -999.f ; + planck_bc1:units = "K" ; + planck_bc1:coordinates = "band_id band_wavelength" ; + float planck_bc2 ; + planck_bc2:long_name = "spectral bandpass correction scale factor for brightness temperature (B(nu) - bc_1)/bc_2 where B()=planck_function() and nu=wavenumber" ; + planck_bc2:_FillValue = -999.f ; + planck_bc2:units = "1" ; + planck_bc2:coordinates = "band_id band_wavelength" ; + int valid_pixel_count ; + valid_pixel_count:long_name = "number of good and conditionally usable pixels" ; + valid_pixel_count:_FillValue = -1 ; + valid_pixel_count:units = "count" ; + valid_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ; + valid_pixel_count:grid_mapping = "goes_imager_projection" ; + valid_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000028 rad comment: good and conditionally usable quality pixels only)" ; + int missing_pixel_count ; + missing_pixel_count:long_name = "number of missing pixels" ; + missing_pixel_count:_FillValue = -1 ; + missing_pixel_count:units = "count" ; + missing_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ; + missing_pixel_count:grid_mapping = "goes_imager_projection" ; + missing_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000028 rad comment: missing ABI fixed grid pixels only)" ; + int saturated_pixel_count ; + saturated_pixel_count:long_name = "number of saturated pixels" ; + saturated_pixel_count:_FillValue = -1 ; + saturated_pixel_count:units = "count" ; + saturated_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ; + saturated_pixel_count:grid_mapping = "goes_imager_projection" ; + saturated_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000028 rad comment: radiometrically saturated geolocated/not missing pixels only)" ; + int undersaturated_pixel_count ; + undersaturated_pixel_count:long_name = "number of undersaturated pixels" ; + undersaturated_pixel_count:_FillValue = -1 ; + undersaturated_pixel_count:units = "count" ; + undersaturated_pixel_count:coordinates = "band_id band_wavelength t y_image x_image" ; + undersaturated_pixel_count:grid_mapping = "goes_imager_projection" ; + undersaturated_pixel_count:cell_methods = "t: sum area: sum (interval: 0.000028 rad comment: radiometrically undersaturated geolocated/not missing pixels only)" ; + int focal_plane_temperature_threshold_exceeded_count ; + focal_plane_temperature_threshold_exceeded_count:long_name = "number of pixels whose temperatures exceeded the threshold" ; + focal_plane_temperature_threshold_exceeded_count:_FillValue = -1 ; + focal_plane_temperature_threshold_exceeded_count:units = "count" ; + focal_plane_temperature_threshold_exceeded_count:coordinates = "band_id band_wavelength t y_image x_image" ; + focal_plane_temperature_threshold_exceeded_count:grid_mapping = "goes_imager_projection" ; + focal_plane_temperature_threshold_exceeded_count:cell_methods = "t: sum area: sum (interval: 0.000028 rad comment: temperature exceeded pixels only)" ; + float min_radiance_value_of_valid_pixels ; + min_radiance_value_of_valid_pixels:long_name = "minimum radiance value of pixels" ; + min_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavelength" ; + min_radiance_value_of_valid_pixels:_FillValue = -999.f ; + min_radiance_value_of_valid_pixels:valid_range = -25.93665f, 804.0361f ; + min_radiance_value_of_valid_pixels:units = "W m-2 sr-1 um-1" ; + min_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ; + min_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ; + min_radiance_value_of_valid_pixels:cell_methods = "t: sum area: minimum (interval: 0.000028 rad comment: good and conditionally usable quality pixels only)" ; + float max_radiance_value_of_valid_pixels ; + max_radiance_value_of_valid_pixels:long_name = "maximum radiance value of pixels" ; + max_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavelength" ; + max_radiance_value_of_valid_pixels:_FillValue = -999.f ; + max_radiance_value_of_valid_pixels:valid_range = -25.93665f, 804.0361f ; + max_radiance_value_of_valid_pixels:units = "W m-2 sr-1 um-1" ; + max_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ; + max_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ; + max_radiance_value_of_valid_pixels:cell_methods = "t: sum area: maximum (interval: 0.000028 rad comment: good and conditionally usable quality pixels only)" ; + float mean_radiance_value_of_valid_pixels ; + mean_radiance_value_of_valid_pixels:long_name = "mean radiance value of pixels" ; + mean_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavelength" ; + mean_radiance_value_of_valid_pixels:_FillValue = -999.f ; + mean_radiance_value_of_valid_pixels:valid_range = -25.93665f, 804.0361f ; + mean_radiance_value_of_valid_pixels:units = "W m-2 sr-1 um-1" ; + mean_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ; + mean_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ; + mean_radiance_value_of_valid_pixels:cell_methods = "t: sum area: mean (interval: 0.000028 rad comment: good and conditionally usable quality pixels only)" ; + float std_dev_radiance_value_of_valid_pixels ; + std_dev_radiance_value_of_valid_pixels:long_name = "standard deviation of radiance values of pixels" ; + std_dev_radiance_value_of_valid_pixels:standard_name = "toa_outgoing_radiance_per_unit_wavelength" ; + std_dev_radiance_value_of_valid_pixels:_FillValue = -999.f ; + std_dev_radiance_value_of_valid_pixels:units = "W m-2 sr-1 um-1" ; + std_dev_radiance_value_of_valid_pixels:coordinates = "band_id band_wavelength t y_image x_image" ; + std_dev_radiance_value_of_valid_pixels:grid_mapping = "goes_imager_projection" ; + std_dev_radiance_value_of_valid_pixels:cell_methods = "t: sum area: standard_deviation (interval: 0.000028 rad comment: good and conditionally usable quality pixels only)" ; + float maximum_focal_plane_temperature ; + maximum_focal_plane_temperature:long_name = "maximum focal plane temperature value" ; + maximum_focal_plane_temperature:_FillValue = -999.f ; + maximum_focal_plane_temperature:valid_range = 0.f, 999.f ; + maximum_focal_plane_temperature:units = "K" ; + float focal_plane_temperature_threshold_increasing ; + focal_plane_temperature_threshold_increasing:long_name = "focal plane temperature threshold increasing bounds value" ; + focal_plane_temperature_threshold_increasing:_FillValue = -999.f ; + focal_plane_temperature_threshold_increasing:valid_range = 0.f, 999.f ; + focal_plane_temperature_threshold_increasing:units = "K" ; + float focal_plane_temperature_threshold_decreasing ; + focal_plane_temperature_threshold_decreasing:long_name = "focal plane temperature threshold decreasing bounds value" ; + focal_plane_temperature_threshold_decreasing:_FillValue = -999.f ; + focal_plane_temperature_threshold_decreasing:valid_range = 0.f, 999.f ; + focal_plane_temperature_threshold_decreasing:units = "K" ; + float percent_uncorrectable_L0_errors ; + percent_uncorrectable_L0_errors:long_name = "percent data lost due to uncorrectable L0 errors" ; + percent_uncorrectable_L0_errors:_FillValue = -999.f ; + percent_uncorrectable_L0_errors:valid_range = 0.f, 1.f ; + percent_uncorrectable_L0_errors:units = "percent" ; + percent_uncorrectable_L0_errors:coordinates = "t y_image x_image" ; + percent_uncorrectable_L0_errors:grid_mapping = "goes_imager_projection" ; + percent_uncorrectable_L0_errors:cell_methods = "t: sum area: sum (uncorrectable L0 errors only)" ; + float earth_sun_distance_anomaly_in_AU ; + earth_sun_distance_anomaly_in_AU:long_name = "earth sun distance anomaly in astronomical units" ; + earth_sun_distance_anomaly_in_AU:_FillValue = -999.f ; + earth_sun_distance_anomaly_in_AU:units = "ua" ; + earth_sun_distance_anomaly_in_AU:coordinates = "t" ; + earth_sun_distance_anomaly_in_AU:cell_methods = "t: mean" ; + int algorithm_dynamic_input_data_container ; + algorithm_dynamic_input_data_container:long_name = "container for filenames of dynamic algorithm input data" ; + algorithm_dynamic_input_data_container:input_ABI_L0_data = "OR_ABI-L0-F-M6_G16_s20220011800205_e20220011809513_c*.nc" ; + int processing_parm_version_container ; + processing_parm_version_container:long_name = "container for processing parameter filenames" ; + processing_parm_version_container:L1b_processing_parm_version = "OR-PARM-RAD_G16_v01r00.zip" ; + int algorithm_product_version_container ; + algorithm_product_version_container:long_name = "container for algorithm package filename and product version" ; + algorithm_product_version_container:algorithm_version = "OR_ABI-L1b-ALG-RAD_v01r00.zip" ; + algorithm_product_version_container:product_version = "v01r00" ; + double t_star_look(num_star_looks) ; + t_star_look:long_name = "J2000 epoch time of star observed in seconds" ; + t_star_look:standard_name = "time" ; + t_star_look:units = "seconds since 2000-01-01 12:00:00" ; + t_star_look:axis = "T" ; + float band_wavelength_star_look(num_star_looks) ; + band_wavelength_star_look:long_name = "ABI band central wavelength associated with observed star" ; + band_wavelength_star_look:standard_name = "sensor_band_central_radiation_wavelength" ; + band_wavelength_star_look:units = "um" ; + short star_id(num_star_looks) ; + star_id:long_name = "ABI star catalog identifier associated with observed star" ; + star_id:_Unsigned = "true" ; + star_id:_FillValue = -1s ; + star_id:coordinates = "band_id band_wavelength_star_look t_star_look" ; + int channel_integration_time ; + channel_integration_time:long_name = "Channel-dependent Channel Integration Time, as defined in the VNIR or IR Channel Configuration Table Telemetry" ; + channel_integration_time:_FillValue = -1 ; + channel_integration_time:units = "count" ; + int channel_gain_field ; + channel_gain_field:long_name = "Channel-dependent Gain Field, as defined in the VNIR or IR Channel Configuration Table Telemetry" ; + channel_gain_field:_FillValue = -1 ; + channel_gain_field:units = "1" ; + +// global attributes: + :naming_authority = "gov.nesdis.noaa" ; + :Conventions = "CF-1.7" ; + :standard_name_vocabulary = "CF Standard Name Table (v35, 20 July 2016)" ; + :institution = "DOC/NOAA/NESDIS > U.S. Department of Commerce, National Oceanic and Atmospheric Administration, National Environmental Satellite, Data, and Information Services" ; + :project = "GOES" ; + :production_site = "WCDAS" ; + :production_environment = "OE" ; + :spatial_resolution = "1km at nadir" ; + :Metadata_Conventions = "Unidata Dataset Discovery v1.0" ; + :orbital_slot = "GOES-East" ; + :platform_ID = "G16" ; + :instrument_type = "GOES-R Series Advanced Baseline Imager (ABI)" ; + :scene_id = "Full Disk" ; + :instrument_ID = "FM1" ; + :title = "ABI L1b Radiances" ; + :summary = "Single reflective band ABI L1b Radiance Products are digital maps of outgoing radiance values at the top of the atmosphere for visible and near-IR bands." ; + :keywords = "SPECTRAL/ENGINEERING > VISIBLE WAVELENGTHS > VISIBLE RADIANCE" ; + :keywords_vocabulary = "NASA Global Change Master Directory (GCMD) Earth Science Keywords, Version 7.0.0.0.0" ; + :iso_series_metadata_id = "a70be540-c38b-11e0-962b-0800200c9a66" ; + :license = "Unclassified data. Access is restricted to approved users only." ; + :processing_level = "National Aeronautics and Space Administration (NASA) L1b" ; + :cdm_data_type = "Image" ; + :dataset_name = "OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc" ; + :production_data_source = "Realtime" ; + :timeline_id = "ABI Mode 6" ; + :date_created = "2022-01-01T18:09:56.2Z" ; + :time_coverage_start = "2022-01-01T18:00:20.5Z" ; + :time_coverage_end = "2022-01-01T18:09:51.3Z" ; + :LUT_Filenames = "SpaceLookParams(FM1A_CDRL79RevP_PR_09_00_02)-637827000.0.h5 QTableBand01(FM1A_CDRL79RevH_DO_07_00_00)-582860861.0.h5 CalTargetTimeIntervals(FM1A_CDRL79RevP_DO_08_00_01)-611906620.0.h5 BandSaturationLimits(FM1A_CDRL79RevH_DO_08_00_00)-600000000.0.h5 SolarSpaceLookParams(FM1A_CDRL79RevH_DO_09_00_00)-600765435.0.h5 DeadRowListParams(FM1A_CDRL79RevH_DO_08_00_00)-600000000.0.h5 Mirror_Record(FM1A_CDRL79RevG_DO_07_00_00)-582860861.0.h5 KalmanAstroConsts(FM1A_CDRL79RevH_DO_08_00_00)-600000000.0.xml KalmanFilterControls(FM1A_PR_09_08_02)-677650371.0.xml KalmanMeasMaxSensibles(FMAA_INT_ONLY_DO_09_01_00)-652936814.0.xml KalmanPreprocessorControls(FM1A_CDRL79RevJ_PR_09_06_02)-657795700.0.xml KalmanReferenceData(FM1A_CDRL79RevH_DO_08_00_00)-888.0.xml KalmanStarCatalogs(FM1A_CDRL79RevH_DO_08_00_00)-600000000.0.xml ABI_NavigationRDP_Band01(FM1A_CDRL79RevJ_DO_07_00_00)-582860861.0.xml ABI_NavigationParameters_Band01(FM1A_CDRL79RevH_DO_07_00_00)-582860861.0.xml ABI_ResamplingImplementation_Band01(FM1A_CDRL79RevH_DO_07_02_00)-602129336.0.xml ABI_ResamplingParameters_Band01(FM1A_CDRL79RevJ_DO_07_00_00)-582860861.0.xml StarLookParams(FM1A_CDRL79RevH_DO_08_00_00)-600000000.0.h5 StarDetectionParams(FM1A_CDRL79RevJ_DO_07_00_00)-582860861.0.xml ResamplingScaledConversion(FMAA_INT_ONLY_DO_08_00_00)-1111.0.xml BlockReleaseRegions(FMAA_INT_ONLY_DO_08_00_00)-2222.0.csv VNIR_RetrievalParameters(FM1A_CDRL79RevH_DO_08_00_00)-600000000.0.h5 SCT_Record(FM1A_CDRL79RevM_DO_09_00_00)-600765435.0.h5 ICM_ConversionConsts(FM1A_CDRL43-18_DO_09_01_00)-652936750.0.h5 ICM_SensorCoefficients(FM1A_TMABI_18_159_TMABI_18_533_DO_09_05_00)-676949608.0.h5" ; + :id = "75de858d-c386-4159-a95e-bce8a0d3d61e" ; +} diff --git a/nc_test/ref_tst_http_nc4f.cdl b/nc_test/ref_tst_http_nc4f.cdl new file mode 100644 index 0000000000..e4da5f8a64 --- /dev/null +++ b/nc_test/ref_tst_http_nc4f.cdl @@ -0,0 +1,50 @@ +netcdf HadCRUT.4.6.0.0.median { +dimensions: + latitude = 36 ; + longitude = 72 ; + field_status_string_length = 1 ; + time = UNLIMITED ; // (2062 currently) +variables: + float latitude(latitude) ; + latitude:standard_name = "latitude" ; + latitude:long_name = "latitude" ; + latitude:point_spacing = "even" ; + latitude:units = "degrees_north" ; + latitude:axis = "Y" ; + float longitude(longitude) ; + longitude:standard_name = "longitude" ; + longitude:long_name = "longitude" ; + longitude:point_spacing = "even" ; + longitude:units = "degrees_east" ; + longitude:axis = "X" ; + float time(time) ; + time:standard_name = "time" ; + time:long_name = "time" ; + time:units = "days since 1850-1-1 00:00:00" ; + time:calendar = "gregorian" ; + time:start_year = 1850s ; + time:end_year = 2021s ; + time:start_month = 1s ; + time:end_month = 10s ; + time:axis = "T" ; + float temperature_anomaly(time, latitude, longitude) ; + temperature_anomaly:long_name = "near_surface_temperature_anomaly" ; + temperature_anomaly:units = "K" ; + temperature_anomaly:missing_value = -1.e+30f ; + temperature_anomaly:_FillValue = -1.e+30f ; + temperature_anomaly:reference_period = 1961s, 1990s ; + char field_status(time, field_status_string_length) ; + field_status:long_name = "field_status" ; + +// global attributes: + :title = "HadCRUT4 near-surface temperature ensemble data - ensemble median" ; + :institution = "Met Office Hadley Centre / Climatic Research Unit, University of East Anglia" ; + :history = "Updated at 15/12/2021 19:37:52" ; + :source = "CRUTEM.4.6.0.0, HadSST.3.1.1.0" ; + :comment = "" ; + :reference = "Morice, C. P., J. J. Kennedy, N. A. Rayner, and P. D. Jones (2012), Quantifying uncertainties in global and regional temperature change using an ensemble of observational estimates: The HadCRUT4 dataset, J. Geophys. Res., doi:10.1029/2011JD017187" ; + :version = "HadCRUT.4.6.0.0" ; + :Conventions = "CF-1.0" ; + :ensemble_members = 100s ; + :ensemble_member_index = 0s ; +} diff --git a/nc_test/test_byterange.sh b/nc_test/test_byterange.sh index 8c13cd3480..ee5c567bb2 100755 --- a/nc_test/test_byterange.sh +++ b/nc_test/test_byterange.sh @@ -9,17 +9,25 @@ set -e # Test Urls if test "x$FEATURE_THREDDSTEST" = x1 ; then -URL3="https://thredds-test.unidata.ucar.edu/thredds/fileServer/pointData/cf_dsg/example/point.nc#mode=bytes&aws.profile=none" -URL4b="https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files/METAR_20170910_0000.nc#bytes&aws.profile=none" +URL3="https://thredds-test.unidata.ucar.edu/thredds/fileServer/pointData/cf_dsg/example/point.nc#mode=bytes" +URL4b="https://thredds-test.unidata.ucar.edu/thredds/fileServer/irma/metar/files/METAR_20170910_0000.nc#bytes" fi -URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes&aws.profile=none" -URL4c="s3://noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes&aws.profile=none" +if test "x$FEATURE_S3TESTS" = xyes ; then +URL4a="https://s3.us-east-1.amazonaws.com/noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes" +URL4c="s3://noaa-goes16/ABI-L1b-RadC/2017/059/03/OR_ABI-L1b-RadC-M3C13_G16_s20170590337505_e20170590340289_c20170590340316.nc#mode=bytes" +# Requires auth +URL3b="s3://unidata-zarr-test-data/byterangefiles/upload3.nc#bytes" # Requires auth -URL3b="s3://unidata-zarr-test-data/byterangefiles/upload3.nc#bytes&aws.profile=unidata" URL4d="s3://unidata-zarr-test-data/byterangefiles/upload4.nc#bytes&aws.profile=unidata" +# Test alternate URL with no specified region +URL4e="http://noaa-goes16.s3.amazonaws.com/ABI-L1b-RadF/2022/001/18/OR_ABI-L1b-RadF-M6C01_G16_s20220011800205_e20220011809513_c20220011809562.nc#mode=bytes,s3" +fi +URL4f="https://crudata.uea.ac.uk/cru/data/temperature/HadCRUT.4.6.0.0.median.nc#mode=bytes" + # Do not use unless we know it has some permanence (note the segment 'testing' in the URL); URL4x="https://s3.us-west-2.amazonaws.com/coawst-public/testing/HadCRUT.4.6.0.0.median.nc#mode=bytes,&aws.profile=none" + echo "" testsetup() { @@ -90,22 +98,27 @@ if test "x$FEATURE_THREDDSTEST" = x1 ; then fi if test "x$FEATURE_HDF5" = xyes ; then + echo "***Test remote netcdf-4 files: non-s3" if test "x$FEATURE_THREDDSTEST" = x1 ; then - echo "***Test remote netcdf-4 file: non-s3" testbytes nc4b netCDF-4 "$URL4b" fi + testbytes nc4f netCDF-4 "$URL4f" +fi + +if test "x$FEATURE_S3" = xyes ; then + echo "***Test remote netcdf-3 file: s3 auth" + tests3auth nc3b classic "$URL3b" fi -if test "x$FEATURE_S3TESTS" = xyes && test "x$FEATURE_HDF5" = xyes ; then + +if test "x$FEATURE_S3" = xyes && test "x$FEATURE_HDF5" = xyes ; then echo "***Test remote netdf-4 file: s3" testbytes nc4a netCDF-4 "$URL4a" echo "***Test remote netcdf-4 file: s3" testbytes nc4c netCDF-4 "$URL4c" echo "***Test remote netcdf-4 file: s3 auth" tests3auth nc4d netCDF-4 "$URL4d" -fi -if test "x$FEATURE_S3TESTS" = xyes ; then - echo "***Test remote netcdf-3 file: s3 auth" - tests3auth nc3b classic "$URL3b" + echo "***Test remote netcdf-4 file: s3 noauth" + testbytes nc4e netCDF-4 "$URL4e" fi # Cleanup diff --git a/nc_test4/tst_quantize.c b/nc_test4/tst_quantize.c index 2619577d81..6a3f712144 100644 --- a/nc_test4/tst_quantize.c +++ b/nc_test4/tst_quantize.c @@ -515,6 +515,7 @@ main(int argc, char **argv) union DU dfin; union DU dfout; union DU double_xpect[NUM_QUANTIZE_MODES][DIM_LEN_5]; + NC_UNUSED(fout); NC_UNUSED(dfout); switch (quantize_mode[q]) { @@ -633,6 +634,7 @@ main(int argc, char **argv) union FU fout; union DU dfout; int nsd_att_in; + NC_UNUSED(fout); NC_UNUSED(dfout); /* Open the file and check metadata. */ if (nc_open(FILE_NAME, NC_WRITE, &ncid)) ERR; @@ -752,6 +754,7 @@ main(int argc, char **argv) union FU fout; union DU dfout; union DU double_xpect[NUM_QUANTIZE_MODES][DIM_LEN_5]; + NC_UNUSED(fout); NC_UNUSED(dfout); switch (quantize_mode[q]) { @@ -875,6 +878,8 @@ main(int argc, char **argv) union FU fout; union DU dfout; union DU double_xpect[NUM_QUANTIZE_MODES][DIM_LEN_5]; + NC_UNUSED(fout); NC_UNUSED(dfout); + switch (quantize_mode[q]) { case NC_QUANTIZE_BITGROOM: @@ -1003,6 +1008,8 @@ main(int argc, char **argv) union FU fout; union DU dfout; union DU double_xpect[NUM_QUANTIZE_MODES][DIM_LEN_5]; + NC_UNUSED(fout); NC_UNUSED(dfout); + switch (quantize_mode[q]) { case NC_QUANTIZE_BITGROOM: diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index f505bc2642..afa27ac3e5 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -85,7 +85,7 @@ IF(ENABLE_TESTS) TARGET_INCLUDE_DIRECTORIES(zisjson PUBLIC ../libnczarr) BUILD_BIN_TEST(zs3parse ${COMMONSRC}) TARGET_INCLUDE_DIRECTORIES(zs3parse PUBLIC ../libnczarr) - if(ENABLE_NCZARR_S3) + if(ENABLE_S3) BUILD_BIN_TEST(s3util ${COMMONSRC}) TARGET_INCLUDE_DIRECTORIES(s3util PUBLIC ../libnczarr) endif() @@ -124,7 +124,7 @@ IF(ENABLE_TESTS) BUILD_BIN_TEST(test_quantize ${TSTCOMMONSRC}) add_sh_test(nczarr_test run_quantize) - if(ENABLE_NCZARR_S3) + if(ENABLE_S3) add_sh_test(nczarr_test run_s3_cleanup) ENDIF() diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index 5e12376997..12416c747d 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -117,7 +117,7 @@ zisjson_SOURCES = zisjson.c noinst_PROGRAMS += zs3parse zs3parse_SOURCES = zs3parse.c -if ENABLE_NCZARR_S3 +if ENABLE_S3 noinst_PROGRAMS += s3util s3util_SOURCES = s3util.c TESTS += run_s3_cleanup.sh diff --git a/nczarr_test/run_interop.sh b/nczarr_test/run_interop.sh index f3d7812d84..22baf1ebf4 100755 --- a/nczarr_test/run_interop.sh +++ b/nczarr_test/run_interop.sh @@ -14,6 +14,7 @@ set -e UH="${NCZARR_S3_TEST_HOST}" UB="${NCZARR_S3_TEST_BUCKET}" +RESULTSDIR="${builddir}/tmp_interop" testcasefile() { zext=file @@ -21,7 +22,7 @@ testcasefile() { mode=$2 metaonly=$3 if test "x$metaonly" = xmetaonly ; then flags="-h"; fi - fileargs ${builddir}/ref_$base "mode=$mode,$zext" + fileargs ${RESULTSDIR}/ref_$base "mode=$mode,$zext" rm -f tmp_${base}_${zext}.cdl ${NCDUMP} $flags $fileurl > tmp_${base}_${zext}.cdl diff -b ${srcdir}/ref_${base}.cdl tmp_${base}_${zext}.cdl @@ -31,7 +32,7 @@ testcasezip() { zext=zip base=$1 mode=$2 - fileargs ${builddir}/ref_$base "mode=$mode,$zext" + fileargs ${RESULTSDIR}/ref_$base "mode=$mode,$zext" rm -f tmp_${base}_${zext}.cdl ${NCDUMP} -h $flags $fileurl > tmp_${base}_${zext}.cdl diff -b ${srcdir}/ref_${base}.cdl tmp_${base}_${zext}.cdl @@ -45,7 +46,7 @@ testcases3() { url="https://${UH}/${UB}/${base}.zarr#mode=${mode},s3" ${NCDUMP} $url > tmp_${base}_${zext}.cdl # Find the proper ref file - diff -b ${builddir}/ref_${base}.cdl tmp_${base}_${zext}.cdl + diff -b ${RESULTSDIR}/ref_${base}.cdl tmp_${base}_${zext}.cdl } testallcases() { @@ -53,47 +54,39 @@ zext=$1 case "$zext" in file) # need to unpack - rm -fr ref_power_901_constants ref_power_901_constants.file - if ! test -f ${builddir}/ref_power_901_constants_orig.zip ; then - cp -f ${srcdir}/ref_power_901_constants_orig.zip ${builddir}/ref_power_901_constants_orig.zip - fi - unzip ${builddir}/ref_power_901_constants_orig.zip > /dev/null - mv ${builddir}/ref_power_901_constants ${builddir}/ref_power_901_constants.file + unzip ${srcdir}/ref_power_901_constants_orig.zip > /dev/null + mv ${RESULTSDIR}/ref_power_901_constants ${RESULTSDIR}/ref_power_901_constants.file testcasefile power_901_constants zarr metaonly; # test xarray as default ;; zip) # Move into position - if ! test -f ${builddir}/ref_power_901_constants.zip ; then - cp -f ${srcdir}/ref_power_901_constants_orig.zip ${builddir}/ref_power_901_constants.zip - fi - if ! test -f ${builddir}/ref_quotes.zip ; then - cp -f ${srcdir}/ref_quotes_orig.zip ${builddir}/ref_quotes.zip - fi + cp -f ${srcdir}/ref_power_901_constants_orig.zip ${RESULTSDIR}/ref_power_901_constants.zip + cp -f ${srcdir}/ref_quotes_orig.zip ${RESULTSDIR}/ref_quotes.zip testcasezip power_901_constants xarray metaonly # Test large constant interoperability testcasezip quotes zarr metaonly ;; s3) # Read a test case created by netcdf-java zarr. - # Move into position - rm -f ${builddir}/ref_zarr_test_data.cdl - if ! test -f ${builddir}/ref_zarr_test_data.cdl.gz ; then - cp -f ${srcdir}/ref_zarr_test_data.cdl.gz ${builddir}/ref_zarr_test_data.cdl.gz - fi + # unpack # Use gunzip because it always appears to be available - gunzip -c ${builddir}/ref_zarr_test_data.cdl.gz > ${builddir}/ref_zarr_test_data.cdl + gunzip -c ${srcdir}/ref_zarr_test_data.cdl.gz > ${RESULTSDIR}/ref_zarr_test_data.cdl testcases3 zarr_test_data xarray ;; *) echo "unimplemented kind: $1" ; exit 1;; esac } +THISDIR=`pwd` +rm -fr ${RESULTSDIR} +mkdir -p ${RESULTSDIR} +cd ${RESULTSDIR} testallcases file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testallcases zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testallcases s3; fi +cd ${THISDIR} # Cleanup -rm -fr ${builddir}/ref_power_901_constants_orig.zip -rm -fr ${builddir}/ref_zarr_test_data.cdl.gz +rm -fr ${RESULTSDIR} exit 0 diff --git a/nczarr_test/run_scalar.sh b/nczarr_test/run_scalar.sh index 5b8315bb0e..694c513062 100755 --- a/nczarr_test/run_scalar.sh +++ b/nczarr_test/run_scalar.sh @@ -7,7 +7,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi # This shell script tests support for the NC_STRING type -set -x set -e zarrscalar() { diff --git a/test_common.in b/test_common.in index 1df44654b2..95d209621a 100644 --- a/test_common.in +++ b/test_common.in @@ -30,11 +30,14 @@ FEATURE_PARALLEL=@HAS_PARALLEL@ # Define selected features of the build FEATURE_HDF5=@HAS_HDF5@ +FEATURE_FILTERTESTS=@DO_FILTER_TESTS@ +FEATURE_PLUGIN_INSTALL_DIR=@PLUGIN_INSTALL_DIR@ +FEATURE_BYTERANGE=@HAS_BYTERANGE@ +FEATURE_S3_SDK=@HAS_S3_SDK@ +FEATURE_S3=@HAS_S3@ FEATURE_NCZARR=@HAS_NCZARR@ FEATURE_S3TESTS=@DO_NCZARR_S3_TESTS@ FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@ -FEATURE_FILTERTESTS=@DO_FILTER_TESTS@ -FEATURE_PLUGIN_INSTALL_DIR=@PLUGIN_INSTALL_DIR@ # Thredds-test server is currently disabled #FEATURE_THREDDSTEST=1