diff --git a/autotest/gcore/vsicurl.py b/autotest/gcore/vsicurl.py index 099da1e57c5c..cd40d72aa5c7 100755 --- a/autotest/gcore/vsicurl.py +++ b/autotest/gcore/vsicurl.py @@ -1288,3 +1288,89 @@ def test_vsicurl_404_repeated_same_resource(server): with pytest.raises(Exception, match="404"): gdal.Open("/vsicurl/http://localhost:%d/does/not/exist.bin" % server.port) + + +############################################################################### + + +@gdaltest.enable_exceptions() +def test_vsicurl_cache_control_not_set(server): + + gdal.VSICurlClearCache() + + handler = webserver.SequentialHandler() + handler.add("GET", "/", 404) + handler.add("HEAD", "/test.txt", 200, {"Content-Length": "3"}) + handler.add("GET", "/test.txt", 200, {}, "foo") + with webserver.install_http_handler(handler): + f = gdal.VSIFOpenL( + "/vsicurl/http://localhost:%d/test.txt" % server.port, + "rb", + ) + assert f is not None + data = gdal.VSIFReadL(1, 3, f).decode("ascii") + gdal.VSIFCloseL(f) + assert data == "foo" + + with webserver.install_http_handler(webserver.SequentialHandler()): + f = gdal.VSIFOpenL( + "/vsicurl/http://localhost:%d/test.txt" % server.port, + "rb", + ) + assert f is not None + data = gdal.VSIFReadL(1, 3, f).decode("ascii") + gdal.VSIFCloseL(f) + assert data == "foo" + + +############################################################################### + + +@gdaltest.enable_exceptions() +def test_vsicurl_cache_control_no_cache(server): + + gdal.VSICurlClearCache() + + handler = webserver.SequentialHandler() + handler.add("GET", "/", 404) + handler.add( + "HEAD", "/test.txt", 200, {"Content-Length": "3", "Cache-Control": "no-cache"} + ) + handler.add( + "GET", + "/test.txt", + 200, + {"Content-Length": "3", "Cache-Control": "no-cache"}, + "foo", + ) + with webserver.install_http_handler(handler): + f = gdal.VSIFOpenL( + "/vsicurl/http://localhost:%d/test.txt" % server.port, + "rb", + ) + assert f is not None + data = gdal.VSIFReadL(1, 3, f).decode("ascii") + gdal.VSIFCloseL(f) + assert data == "foo" + + handler = webserver.SequentialHandler() + handler.add("GET", "/", 404) + handler.add( + "HEAD", "/test.txt", 200, {"Content-Length": "6", "Cache-Control": "no-cache"} + ) + handler.add( + "GET", + "/test.txt", + 200, + {"Content-Length": "6", "Cache-Control": "no-cache"}, + "barbaz", + ) + with webserver.install_http_handler(handler): + f = gdal.VSIFOpenL( + "/vsicurl/http://localhost:%d/test.txt" % server.port, + "rb", + ) + assert f is not None + data = gdal.VSIFReadL(1, 6, f).decode("ascii") + gdal.VSIFCloseL(f) + assert data == "barbaz" diff --git a/autotest/gcore/vsis3.py b/autotest/gcore/vsis3.py index fbb090749aa1..2392acdaecda 100755 --- a/autotest/gcore/vsis3.py +++ b/autotest/gcore/vsis3.py @@ -2199,6 +2199,112 @@ def method(request): assert gdal.GetLastErrorMsg() == "" +############################################################################### +# Test that PUT invalidates cached data + + +def test_vsis3_put_invalidate(aws_test_config, webserver_port): + + gdal.VSICurlClearCache() + + handler = webserver.SequentialHandler() + handler.add("GET", "/s3_fake_bucket3/?delimiter=%2F", 200) + handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"foo") + handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"foo") + + with webserver.install_http_handler(handler): + f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb") + assert f is not None + try: + assert gdal.VSIFReadL(3, 1, f) == b"foo" + finally: + gdal.VSIFCloseL(f) + + handler = webserver.SequentialHandler() + with webserver.install_http_handler(handler): + f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb") + assert f is not None + try: + assert gdal.VSIFReadL(3, 1, f) == b"foo" + finally: + gdal.VSIFCloseL(f) + + handler = webserver.SequentialHandler() + handler.add("PUT", "/s3_fake_bucket3/test_put_invalidate.bin", 200) + with webserver.install_http_handler(handler): + f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "wb") + assert f is not None + try: + assert gdal.VSIFWriteL("barbaw", 1, 6, f) == 6 + finally: + gdal.VSIFCloseL(f) + + handler = webserver.SequentialHandler() + handler.add("GET", "/s3_fake_bucket3/?delimiter=%2F", 200) + handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"barbaw") + handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"barbaw") + + with webserver.install_http_handler(handler): + f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb") + assert f is not None + try: + assert gdal.VSIFReadL(6, 1, f) == b"barbaw" + finally: + gdal.VSIFCloseL(f) + + +############################################################################### +# Test that CopyFile invalidates cached data + + +def test_vsis3_copy_invalidate(aws_test_config, webserver_port, tmp_vsimem): + + gdal.VSICurlClearCache() + + handler = webserver.SequentialHandler() + handler.add("GET", "/s3_fake_bucket3/?delimiter=%2F", 200) + handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"foo") + handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"foo") + + with webserver.install_http_handler(handler): + f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb") + assert f is not None + try: + assert gdal.VSIFReadL(3, 1, f) == b"foo" + finally: + gdal.VSIFCloseL(f) + + handler = webserver.SequentialHandler() + with webserver.install_http_handler(handler): + f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb") + assert f is not None + try: + assert gdal.VSIFReadL(3, 1, f) == b"foo" + finally: + gdal.VSIFCloseL(f) + + handler = webserver.SequentialHandler() + handler.add("PUT", "/s3_fake_bucket3/test_put_invalidate.bin", 200) + memfilename = str(tmp_vsimem / "tmp.bin") + with webserver.install_http_handler(handler), gdaltest.tempfile( + memfilename, b"barbaw" + ): + gdal.CopyFile(memfilename, "/vsis3/s3_fake_bucket3/test_put_invalidate.bin") + + handler = webserver.SequentialHandler() + handler.add("GET", "/s3_fake_bucket3/?delimiter=%2F", 200) + handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"barbaw") + handler.add("GET", "/s3_fake_bucket3/test_put_invalidate.bin", 200, {}, b"barbaw") + + with webserver.install_http_handler(handler): + f = gdal.VSIFOpenL("/vsis3/s3_fake_bucket3/test_put_invalidate.bin", "rb") + assert f is not None + try: + assert gdal.VSIFReadL(6, 1, f) == b"barbaw" + finally: + gdal.VSIFCloseL(f) + + ############################################################################### # Test simple PUT support with retry logic diff --git a/port/cpl_vsil_curl.cpp b/port/cpl_vsil_curl.cpp index 2308a26876c2..237c7eb17319 100644 --- a/port/cpl_vsil_curl.cpp +++ b/port/cpl_vsil_curl.cpp @@ -1363,7 +1363,6 @@ vsi_l_offset VSICurlHandle::GetFileSizeOrHeaders(bool bSetError, } } - if (bGetHeaders) { char **papszHeaders = CSLTokenizeString2(sWriteFuncHeaderData.pBuffer, "\r\n", 0); @@ -1374,7 +1373,17 @@ vsi_l_offset VSICurlHandle::GetFileSizeOrHeaders(bool bSetError, CPLParseNameValue(papszHeaders[i], &pszKey); if (pszKey && pszValue) { - m_aosHeaders.SetNameValue(pszKey, pszValue); + if (bGetHeaders) + { + m_aosHeaders.SetNameValue(pszKey, pszValue); + } + if (EQUAL(pszKey, "Cache-Control") && + EQUAL(pszValue, "no-cache") && + CPLTestBool(CPLGetConfigOption( + "CPL_VSIL_CURL_HONOR_CACHE_CONTROL", "YES"))) + { + m_bCached = false; + } } CPLFree(pszKey); }