Merge pull request OSGeo#10771 from rouault/invisible_vsimem

Add VSIMemGenerateHiddenFilename() and use it in drivers

apps/gdalwarp_lib.cpp

@@ -1175,8 +1175,8 @@ static bool DealWithCOGOptions(CPLStringList &aosCreateOptions, int nSrcCount,
 
     GDALWarpAppOptions oClonedOptions(*psOptions);
     oClonedOptions.bQuiet = true;
-    CPLString osTmpFilename;
-    osTmpFilename.Printf("/vsimem/gdalwarp/%p.tif", &oClonedOptions);
+    const CPLString osTmpFilename(
+        VSIMemGenerateHiddenFilename("gdalwarp_tmp.tif"));
     CPLStringList aosTmpGTiffCreateOptions;
     aosTmpGTiffCreateOptions.SetNameValue("SPARSE_OK", "YES");
     aosTmpGTiffCreateOptions.SetNameValue("TILED", "YES");

autotest/cpp/test_cpl.cpp

@@ -5331,4 +5331,194 @@ TEST_F(test_cpl, CPLSpawn)
 }
 #endif
 
+static bool ENDS_WITH(const char *pszStr, const char *pszEnd)
+{
+    return strlen(pszStr) >= strlen(pszEnd) &&
+           strcmp(pszStr + strlen(pszStr) - strlen(pszEnd), pszEnd) == 0;
+}
+
+TEST_F(test_cpl, VSIMemGenerateHiddenFilename)
+{
+    {
+        // Initial cleanup
+        VSIRmdirRecursive("/vsimem/");
+        VSIRmdirRecursive("/vsimem/.#!HIDDEN!#.");
+
+        // Generate unlisted filename
+        const std::string osFilename1 = VSIMemGenerateHiddenFilename(nullptr);
+        const char *pszFilename1 = osFilename1.c_str();
+        EXPECT_TRUE(STARTS_WITH(pszFilename1, "/vsimem/.#!HIDDEN!#./"));
+        EXPECT_TRUE(ENDS_WITH(pszFilename1, "/unnamed"));
+
+        {
+            // Check the file doesn't exist yet
+            VSIStatBufL sStat;
+            EXPECT_EQ(VSIStatL(pszFilename1, &sStat), -1);
+        }
+
+        // Create the file with some content
+        GByte abyDummyData[1] = {0};
+        VSIFCloseL(VSIFileFromMemBuffer(pszFilename1, abyDummyData,
+                                        sizeof(abyDummyData), false));
+
+        {
+            // Check the file exists now
+            VSIStatBufL sStat;
+            EXPECT_EQ(VSIStatL(pszFilename1, &sStat), 0);
+        }
+
+        // Get's back content
+        EXPECT_EQ(VSIGetMemFileBuffer(pszFilename1, nullptr, false),
+                  abyDummyData);
+
+        {
+            // Check the hidden file doesn't popup
+            const CPLStringList aosFiles(VSIReadDir("/vsimem/"));
+            EXPECT_EQ(aosFiles.size(), 0);
+        }
+
+        {
+            // Check that we can list the below directory if we know it exists
+            // and there's just one subdir
+            const CPLStringList aosFiles(VSIReadDir("/vsimem/.#!HIDDEN!#."));
+            EXPECT_EQ(aosFiles.size(), 1);
+        }
+
+        {
+            // but that it is not an explicit directory
+            VSIStatBufL sStat;
+            EXPECT_EQ(VSIStatL("/vsimem/.#!HIDDEN!#.", &sStat), -1);
+        }
+
+        // Creates second file
+        const std::string osFilename2 = VSIMemGenerateHiddenFilename(nullptr);
+        const char *pszFilename2 = osFilename2.c_str();
+        EXPECT_TRUE(strcmp(pszFilename1, pszFilename2) != 0);
+
+        // Create it
+        VSIFCloseL(VSIFileFromMemBuffer(pszFilename2, abyDummyData,
+                                        sizeof(abyDummyData), false));
+
+        {
+            // Check that we can list the root hidden dir if we know it exists
+            const CPLStringList aosFiles(VSIReadDir("/vsimem/.#!HIDDEN!#."));
+            EXPECT_EQ(aosFiles.size(), 2);
+        }
+
+        {
+            // Create an explicit subdirectory in a hidden directory
+            const std::string osBaseName =
+                VSIMemGenerateHiddenFilename(nullptr);
+            const std::string osSubDir =
+                CPLFormFilename(osBaseName.c_str(), "mysubdir", nullptr);
+            EXPECT_EQ(VSIMkdir(osSubDir.c_str(), 0), 0);
+
+            // Check the subdirectory exists
+            {
+                VSIStatBufL sStat;
+                EXPECT_EQ(VSIStatL(osSubDir.c_str(), &sStat), 0);
+            }
+
+            // but not its hidden parent
+            {
+                VSIStatBufL sStat;
+                EXPECT_EQ(VSIStatL(osBaseName.c_str(), &sStat), -1);
+            }
+
+            // Create file within the subdirectory
+            VSIFCloseL(VSIFileFromMemBuffer(
+                CPLFormFilename(osSubDir.c_str(), "my.bin", nullptr),
+                abyDummyData, sizeof(abyDummyData), false));
+
+            {
+                // Check that we can list the subdirectory
+                const CPLStringList aosFiles(VSIReadDir(osSubDir.c_str()));
+                EXPECT_EQ(aosFiles.size(), 1);
+            }
+
+            {
+                // Check that we can list the root hidden dir if we know it exists
+                const CPLStringList aosFiles(
+                    VSIReadDir("/vsimem/.#!HIDDEN!#."));
+                EXPECT_EQ(aosFiles.size(), 3);
+            }
+        }
+
+        // Directly create a directory with the return of VSIMemGenerateHiddenFilename()
+        {
+            const std::string osDirname = VSIMemGenerateHiddenFilename(nullptr);
+            EXPECT_EQ(VSIMkdir(osDirname.c_str(), 0), 0);
+
+            // Check the subdirectory exists
+            {
+                VSIStatBufL sStat;
+                EXPECT_EQ(VSIStatL(osDirname.c_str(), &sStat), 0);
+            }
+
+            // Create file within the subdirectory
+            VSIFCloseL(VSIFileFromMemBuffer(
+                CPLFormFilename(osDirname.c_str(), "my.bin", nullptr),
+                abyDummyData, sizeof(abyDummyData), false));
+
+            {
+                // Check there's a file in this subdirectory
+                const CPLStringList aosFiles(VSIReadDir(osDirname.c_str()));
+                EXPECT_EQ(aosFiles.size(), 1);
+            }
+
+            EXPECT_EQ(VSIRmdirRecursive(osDirname.c_str()), 0);
+
+            {
+                // Check there's no longer any file in this subdirectory
+                const CPLStringList aosFiles(VSIReadDir(osDirname.c_str()));
+                EXPECT_EQ(aosFiles.size(), 0);
+            }
+
+            {
+                // Check that it no longer exists
+                VSIStatBufL sStat;
+                EXPECT_EQ(VSIStatL(osDirname.c_str(), &sStat), -1);
+            }
+        }
+
+        // Check that operations on "/vsimem/" do not interfere with hidden files
+        {
+            // Create regular file
+            VSIFCloseL(VSIFileFromMemBuffer("/vsimem/regular_file",
+                                            abyDummyData, sizeof(abyDummyData),
+                                            false));
+
+            // Check it is visible
+            EXPECT_EQ(CPLStringList(VSIReadDir("/vsimem/")).size(), 1);
+
+            // Clean root /vsimem/
+            VSIRmdirRecursive("/vsimem/");
+
+            // No more user files
+            EXPECT_TRUE(CPLStringList(VSIReadDir("/vsimem/")).empty());
+
+            // But still hidden files
+            EXPECT_TRUE(
+                !CPLStringList(VSIReadDir("/vsimem/.#!HIDDEN!#.")).empty());
+        }
+
+        // Clean-up hidden files
+        EXPECT_EQ(VSIRmdirRecursive("/vsimem/.#!HIDDEN!#."), 0);
+
+        {
+            // Check the root hidden dir is empty
+            const CPLStringList aosFiles(VSIReadDir("/vsimem/.#!HIDDEN!#."));
+            EXPECT_TRUE(aosFiles.empty());
+        }
+
+        EXPECT_EQ(VSIRmdirRecursive("/vsimem/.#!HIDDEN!#."), 0);
+    }
+
+    {
+        const std::string osFilename = VSIMemGenerateHiddenFilename("foo.bar");
+        const char *pszFilename = osFilename.c_str();
+        EXPECT_TRUE(STARTS_WITH(pszFilename, "/vsimem/.#!HIDDEN!#./"));
+        EXPECT_TRUE(ENDS_WITH(pszFilename, "/foo.bar"));
+    }
+}
 }  // namespace

autotest/gcore/vsifile.py

@@ -150,6 +150,26 @@ def vsifile_generic(filename, options=[]):
 
     gdal.Unlink(filename)
 
+    if not filename.startswith("/vsicrypt/"):
+        assert gdal.RmdirRecursive(filename + "/i_dont_exist") == -1
+
+        subdir = filename + "/subdir"
+        assert gdal.MkdirRecursive(subdir + "/subsubdir", 0o755) == 0
+
+        assert gdal.VSIStatL(subdir) is not None
+        assert gdal.VSIStatL(subdir + "/subsubdir") is not None
+
+        if not filename.startswith("/vsimem/"):
+            assert gdal.Rmdir(subdir) == -1
+            assert gdal.VSIStatL(subdir) is not None
+
+        # Safety belt...
+        assert filename.startswith("tmp/") or filename.startswith("/vsimem/")
+        assert gdal.RmdirRecursive(filename) == 0
+
+        assert gdal.VSIStatL(subdir) is None
+        assert gdal.VSIStatL(subdir + "/subsubdir") is None
+
 
 ###############################################################################
 # Test /vsimem

autotest/gdrivers/jp2openjpeg.py

@@ -2744,10 +2744,10 @@ def test_jp2openjpeg_45():
     )
     del out_ds
 
-    dircontent = gdal.ReadDir("/vsimem/")
+    dircontent = gdal.ReadDir("/vsimem/.#!HIDDEN!#.")
     if dircontent:
         for filename in dircontent:
-            assert not filename.startswith("gmljp2")
+            assert "gmljp2" not in filename
 
     ds = ogr.Open("/vsimem/jp2openjpeg_45.jp2")
     assert ds.GetLayerCount() == 1

autotest/ogr/ogr_csw.py

@@ -46,9 +46,14 @@ def module_disable_exceptions():
 ###############################################################################
 @pytest.fixture(autouse=True, scope="module")
 def setup_and_cleanup():
+
+    vsimem_hidden_before = gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.")
+
     with gdal.config_option("CPL_CURL_ENABLE_VSIMEM", "YES"):
         yield
 
+    assert gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.") == vsimem_hidden_before
+
 
 ###############################################################################
 # Test underlying OGR drivers

autotest/ogr/ogr_wfs.py

@@ -63,9 +63,13 @@ def ogr_wfs_init():
     if gml_ds is None:
         pytest.skip("cannot read GML files")
 
+    vsimem_hidden_before = gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.")
+
     with gdal.config_option("CPL_CURL_ENABLE_VSIMEM", "YES"):
         yield
 
+    assert gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.") == vsimem_hidden_before
+
 
 @pytest.fixture(
     params=["NO", None], scope="module", ids=["without-streaming", "with-streaming"]

frmts/cals/calsdataset.cpp

@@ -331,7 +331,7 @@ GDALDataset *CALSDataset::Open(GDALOpenInfo *poOpenInfo)
 
     // Create a TIFF header for a single-strip CCITTFAX4 file.
     poDS->osTIFFHeaderFilename =
-        CPLSPrintf("/vsimem/cals/header_%p.tiff", poDS);
+        VSIMemGenerateHiddenFilename("cals_header.tiff");
     VSILFILE *fp = VSIFOpenL(poDS->osTIFFHeaderFilename, "wb");
     const int nTagCount = 10;
     const int nHeaderSize = 4 + 4 + 2 + nTagCount * 12 + 4;
@@ -359,7 +359,7 @@ GDALDataset *CALSDataset::Open(GDALOpenInfo *poOpenInfo)
 
     // Create a /vsisparse/ description file assembling the TIFF header
     // with the FAX4 codestream that starts at offset 2048 of the CALS file.
-    poDS->osSparseFilename = CPLSPrintf("/vsimem/cals/sparse_%p.xml", poDS);
+    poDS->osSparseFilename = VSIMemGenerateHiddenFilename("cals_sparse.xml");
     fp = VSIFOpenL(poDS->osSparseFilename, "wb");
     CPLAssert(fp);
     VSIFPrintfL(fp,
@@ -473,7 +473,8 @@ GDALDataset *CALSDataset::CreateCopy(const char *pszFilename,
 
     // Write a in-memory TIFF with just the TIFF header to figure out
     // how large it will be.
-    CPLString osTmpFilename(CPLSPrintf("/vsimem/cals/tmp_%p", poSrcDS));
+    const CPLString osTmpFilename(
+        VSIMemGenerateHiddenFilename("tmp_tif_header"));
     char **papszOptions = nullptr;
     papszOptions = CSLSetNameValue(papszOptions, "COMPRESS", "CCITTFAX4");
     papszOptions = CSLSetNameValue(papszOptions, "NBITS", "1");

frmts/daas/daasdataset.cpp

@@ -2420,7 +2420,7 @@ CPLErr GDALDAASRasterBand::GetBlocks(int nBlockXOff, int nBlockYOff,
     }
     else
     {
-        CPLString osTmpMemFile = CPLSPrintf("/vsimem/daas_%p", this);
+        const CPLString osTmpMemFile = VSIMemGenerateHiddenFilename("daas");
         VSIFCloseL(VSIFileFromMemBuffer(
             osTmpMemFile, psResult->pasMimePart[iDataPart].pabyData,
             psResult->pasMimePart[iDataPart].nDataLen, false));

frmts/ecw/ecwdataset.cpp

@@ -2850,10 +2850,11 @@ GDALDataset *ECWDataset::Open(GDALOpenInfo *poOpenInfo, int bIsJPEG2000)
 /*      There are issues at least in the 5.x series.                    */
 /* -------------------------------------------------------------------- */
 #if ECWSDK_VERSION >= 40
+    constexpr const char *szDETECT_BUG_FILENAME =
+        "__detect_ecw_uint32_bug__.j2k";
     if (bIsJPEG2000 && poDS->eNCSRequestDataType == NCSCT_UINT32 &&
         CPLTestBool(CPLGetConfigOption("ECW_CHECK_CORRECT_DECODING", "TRUE")) &&
-        !STARTS_WITH_CI(poOpenInfo->pszFilename,
-                        "/vsimem/detect_ecw_uint32_bug"))
+        strstr(poOpenInfo->pszFilename, szDETECT_BUG_FILENAME) == nullptr)
     {
         static bool bUINT32_Ok = false;
         {
@@ -2878,7 +2879,7 @@ GDALDataset *ECWDataset::Open(GDALOpenInfo *poOpenInfo, int bIsJPEG2000)
                     0xDF, 0xFF, 0x7F, 0x5F, 0xFF, 0xD9};
 
                 const std::string osTmpFilename =
-                    CPLSPrintf("/vsimem/detect_ecw_uint32_bug_%p.j2k", poDS);
+                    VSIMemGenerateHiddenFilename(szDETECT_BUG_FILENAME);
                 VSIFCloseL(VSIFileFromMemBuffer(
                     osTmpFilename.c_str(),
                     const_cast<GByte *>(abyTestUInt32ImageData),

frmts/eeda/eedaidataset.cpp

@@ -433,7 +433,7 @@ bool GDALEEDAIRasterBand::DecodeGDALDataset(const GByte *pabyData, int nDataLen,
 {
     GDALEEDAIDataset *poGDS = reinterpret_cast<GDALEEDAIDataset *>(poDS);
 
-    CPLString osTmpFilename(CPLSPrintf("/vsimem/eeai/%p", this));
+    const CPLString osTmpFilename(VSIMemGenerateHiddenFilename("eedai"));
     VSIFCloseL(VSIFileFromMemBuffer(
         osTmpFilename, const_cast<GByte *>(pabyData), nDataLen, false));
     const char *const apszDrivers[] = {"PNG", "JPEG", "GTIFF", nullptr};

frmts/esric/esric_dataset.cpp

@@ -896,9 +896,7 @@ CPLErr ECBand::IReadBlock(int nBlockXOff, int nBlockYOff, void *pData)
                  GUInt64(size), GUInt64(offset));
         return CE_Failure;
     }
-    CPLString magic;
-    // Should use some sort of unique
-    magic.Printf("/vsimem/esric_%p.tmp", this);
+    const CPLString magic(VSIMemGenerateHiddenFilename("esric.tmp"));
     auto mfh = VSIFileFromMemBuffer(magic.c_str(), fbuffer.data(), size, false);
     VSIFCloseL(mfh);
     // Can't open a raster by handle?