Merge pull request #1634 from NOAA-GSD/ejh_mem_fix

fix leak of HDF5 typeids

RELEASE_NOTES.md

@@ -14,6 +14,7 @@ This file contains a high-level description of this package's evolution. Release
 * [Enhancement] Restore use of szip compression when writing data (including writing in parallel if HDF5 version is 1.10.3 or greater). See [https://github.com/Unidata/netcdf-c/issues/1546].
 * [Enhancement] Enable use of compact storage option for small vars in netCDF/HDF5 files. See [https://github.com/Unidata/netcdf-c/issues/1570].
 * [Enhancement] Updated benchmarking program bm_file.c to better handle very large files. See [https://github.com/Unidata/netcdf-c/issues/1555].
+* [Bug Fix] Fixed problem of growing memory when netCDF-4 files were opened and closed. See [https://github.com/Unidata/netcdf-c/issues/1575 and https://github.com/Unidata/netcdf-c/issues/1571].
 * [Enhancement] Increased size of maximum allowed name in HDF4 files to NC_MAX_NAME. See [https://github.com/Unidata/netcdf-c/issues/1631].
 
 ## 4.7.3 - November 20, 2019

libhdf5/hdf5internal.c

@@ -582,6 +582,17 @@ close_vars(NC_GRP_INFO_T *grp)
             }
         }
 
+        /* Free the HDF5 typeids. */
+        if (var->type_info->rc == 1)
+        {
+            if (((NC_HDF5_TYPE_INFO_T *)(var->type_info->format_type_info))->hdf_typeid &&
+                H5Tclose(((NC_HDF5_TYPE_INFO_T *)(var->type_info->format_type_info))->hdf_typeid) < 0)
+                return NC_EHDFERR;
+            if (((NC_HDF5_TYPE_INFO_T *)(var->type_info->format_type_info))->native_hdf_typeid &&
+                H5Tclose(((NC_HDF5_TYPE_INFO_T *)(var->type_info->format_type_info))->native_hdf_typeid) < 0)
+                return NC_EHDFERR;
+        }
+
         /* Delete any HDF5 dimscale objid information. */
         if (hdf5_var->dimscale_hdf5_objids)
             free(hdf5_var->dimscale_hdf5_objids);

nc_perf/Makefile.am

@@ -17,10 +17,10 @@ TEST_EXTENSIONS = .sh
 AM_LDFLAGS += ${top_builddir}/liblib/libnetcdf.la
 LDADD = ${top_builddir}/liblib/libnetcdf.la
 
-check_PROGRAMS = tst_create_files bm_file tst_chunks3 tst_ar4		\
-tst_ar4_3d tst_ar4_4d bm_many_objs tst_h_many_atts bm_many_atts		\
-tst_files2 tst_files3 tst_mem tst_knmi bm_netcdf4_recs tst_wrf_reads	\
-tst_attsperf bigmeta openbigmeta tst_bm_rando
+check_PROGRAMS = tst_create_files bm_file tst_chunks3 tst_ar4	\
+tst_ar4_3d tst_ar4_4d bm_many_objs tst_h_many_atts bm_many_atts	\
+tst_files2 tst_files3 tst_mem tst_mem1 tst_knmi bm_netcdf4_recs	\
+tst_wrf_reads tst_attsperf bigmeta openbigmeta tst_bm_rando
 
 bm_file_SOURCES = bm_file.c tst_utils.c
 bm_netcdf4_recs_SOURCES = bm_netcdf4_recs.c tst_utils.c
@@ -35,9 +35,9 @@ tst_knmi_SOURCES = tst_knmi.c tst_utils.c
 tst_wrf_reads_SOURCES = tst_wrf_reads.c tst_utils.c
 tst_bm_rando_SOURCES = tst_bm_rando.c tst_utils.c
 
-TESTS = tst_ar4_3d tst_create_files tst_files3 tst_mem run_knmi_bm.sh	\
-tst_wrf_reads tst_attsperf perftest.sh run_tst_chunks.sh		\
-run_bm_elena.sh tst_bm_rando
+TESTS = tst_ar4_3d tst_create_files tst_files3 tst_mem tst_mem1	\
+run_knmi_bm.sh tst_wrf_reads tst_attsperf perftest.sh		\
+run_tst_chunks.sh run_bm_elena.sh tst_bm_rando
 
 run_bm_elena.log: tst_create_files.log
 

nc_perf/tst_mem1.c

@@ -0,0 +1,54 @@
+/* This is part of the netCDF package. Copyright 2020 University
+   Corporation for Atmospheric Research/Unidata See COPYRIGHT file for
+   conditions of use.
+
+   Test internal netcdf-4 file code for memory leaks. This test was
+   suggest by Jeff Whitaker. See
+   https://github.com/Unidata/netcdf-c/issues/1575.
+
+   Ed Hartnett 2/9/20
+*/
+
+#include <config.h>
+#include <nc_tests.h>
+#include <err_macros.h>
+#include <sys/resource.h>
+
+#define FILE_NAME "tst_mem1.nc"
+#define NUM_FILE_OPENS 10000
+
+int main()
+{
+    int ncid, varid, idx;
+    struct rusage r_usage;
+
+    printf("\n*** Testing netcdf-4 memory use.\n");
+    printf("*** testing mem use opening/closing file...");
+    {
+        long my_rss = 0;
+
+        if (nc_create(FILE_NAME, NC_CLOBBER | NC_NETCDF4, &ncid)) ERR;
+        /* if (nc_create(FILE_NAME, NC_CLOBBER, &ncid)) ERR; */
+        if (nc_def_var(ncid, "dummy", NC_DOUBLE, 0, NULL, &varid)) ERR;
+        if (nc_close(ncid)) ERR;
+
+        for (idx = 0; idx < NUM_FILE_OPENS; idx++)
+        {
+            if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
+            if (nc_close(ncid)) ERR;
+            getrusage(RUSAGE_SELF, &r_usage);
+            /* if (!(idx % 100)) */
+            /*     printf("Memory usage: %ld kilobytes\n",r_usage.ru_maxrss); */
+
+            /* Memory usage goes up in the first couple of opens, but
+             * should then remain steady. Check that it does not
+             * change after the first 10 iterations. */
+            if (!my_rss || idx < 10)
+                my_rss = r_usage.ru_maxrss;
+            else
+                if (my_rss != r_usage.ru_maxrss) ERR;
+        };
+    }
+    SUMMARIZE_ERR;
+    FINAL_RESULTS;
+}