Merge pull request #121 from rapidsai/branch-21.10

CHANGELOG.md

@@ -1,3 +1,7 @@
+# cuCIM 21.10.00 (Date TBD)
+
+Please see https://github.com/rapidsai/cucim/releases/tag/v21.10.00a for the latest changes to this development branch.
+
 # cuCIM 21.08.00 (4 Aug 2021)
 
 ## 🐛 Bug Fixes

CMakeLists.txt

@@ -103,7 +103,7 @@ include(ExternalProject)
 ################################################################################
 # Options
 ################################################################################
-option(CUCIM_SUPPORT_GDS "Support cufile library" OFF)
+option(CUCIM_SUPPORT_GDS "Support cufile library" ON)
 option(CUCIM_STATIC_GDS "Use static cufile library" OFF)
 option(CUCIM_SUPPORT_CUDA "Support CUDA" ON)
 

README.md

@@ -22,8 +22,6 @@
 
 `<CUDA version>` should be 11.0+ (e.g., `11.0`, `11.2`, etc.)
 
-**NOTE:** The first cuCIM conda package (v0.19.0) would be available on 4/19/2021.
-
 #### Conda (nightlies)
 
 > conda create -n cucim -c rapidsai-nightly -c conda-forge cucim cudatoolkit=`<CUDA version>`
@@ -53,6 +51,7 @@ docker rm -v ${tmp_id}
 ```
 
 ## Build/Install from Source
+
 See build [instructions](CONTRIBUTING.md#setting-up-your-build-environment).
 
 ## Contributing Guide

VERSION

@@ -1 +1 @@
-21.08.01
+21.10.00

ci/gpu/build.sh

@@ -52,7 +52,7 @@ gpuci_logger "Activate conda env"
 conda activate rapids
 
 gpuci_logger "Install dependencies"
-gpuci_conda_retry install -y -c rapidsai-nightly \
+gpuci_mamba_retry install -y -c rapidsai-nightly \
     "cudatoolkit=${CUDA_VER}.*" \
     "rapids-build-env=$MINOR_VERSION.*"
 
@@ -69,7 +69,7 @@ CUCIM_BLD_PATH=/opt/conda/envs/rapids/conda-bld
 mkdir -p ${CUCIM_BLD_PATH}
 
 
-gpuci_conda_retry build -c ${LIBCUCIM_BLD_PATH} -c conda-forge -c rapidsai-nightly \
+gpuci_mamba_retry build -c ${LIBCUCIM_BLD_PATH} -c conda-forge -c rapidsai-nightly \
     --dirty \
     --no-remove-work-dir \
     --croot ${CUCIM_BLD_PATH} \
@@ -82,7 +82,7 @@ gpuci_conda_retry build -c ${LIBCUCIM_BLD_PATH} -c conda-forge -c rapidsai-night
 
 # Install cuCIM and its dependencies
 gpuci_logger "Installing cuCIM and its dependencies"
-gpuci_conda_retry install -y -c ${LIBCUCIM_BLD_PATH} -c ${CUCIM_BLD_PATH} -c rapidsai-nightly \
+gpuci_mamba_retry install -y -c ${LIBCUCIM_BLD_PATH} -c ${CUCIM_BLD_PATH} -c rapidsai-nightly \
     "rapids-build-env=$MINOR_VERSION.*" \
     libcucim \
     cucim

cpp/CMakeLists.txt

@@ -65,6 +65,7 @@ add_library(${CUCIM_PACKAGE_NAME}
         include/cucim/memory/memory_manager.h
         include/cucim/util/cuda.h
         include/cucim/util/file.h
+        include/cucim/util/platform.h
         include/cucim/3rdparty/dlpack/dlpack.h
         include/cucim/3rdparty/dlpack/dlpackcpp.h
         src/cuimage.cpp
@@ -95,7 +96,8 @@ add_library(${CUCIM_PACKAGE_NAME}
         src/logger/logger.cpp
         src/logger/timer.cpp
         src/memory/memory_manager.cu
-        src/util/file.cpp)
+        src/util/file.cpp
+        src/util/platform.cpp)
 
 # Compile options
 set_target_properties(${CUCIM_PACKAGE_NAME}

cpp/include/cucim/cuimage.h

@@ -164,6 +164,8 @@ class EXPORT_VISIBLE CuImage : public std::enable_shared_from_this<CuImage>
 
     void save(std::string file_path) const;
 
+    void close();
+
 private:
     using Mutex = std::mutex;
     using ScopedLock = std::scoped_lock<Mutex>;

cpp/include/cucim/util/platform.h

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+//
+#ifndef CUCIM_UTIL_PLATFORM_H
+#define CUCIM_UTIL_PLATFORM_H
+
+#include "cucim/macros/api_header.h"
+
+/**
+ * @brief Platform-specific macros and functions.
+ */
+namespace cucim::util
+{
+
+EXPORT_VISIBLE bool is_in_wsl();
+
+} // namespace cucim::util
+
+#endif // CUCIM_UTIL_PLATFORM_H

cpp/plugins/cucim.kit.cuslide/CMakeLists.txt

@@ -172,6 +172,8 @@ add_library(${CUCIM_PLUGIN_NAME}
     src/cuslide/deflate/deflate.h
     src/cuslide/jpeg/libjpeg_turbo.cpp
     src/cuslide/jpeg/libjpeg_turbo.h
+    src/cuslide/raw/raw.cpp
+    src/cuslide/raw/raw.h
     src/cuslide/tiff/ifd.cpp
     src/cuslide/tiff/ifd.h
     src/cuslide/tiff/tiff.cpp

cpp/plugins/cucim.kit.cuslide/VERSION

@@ -1 +1 @@
-21.08.01
+21.10.00

cpp/plugins/cucim.kit.cuslide/src/cuslide/deflate/deflate.cpp

@@ -40,6 +40,20 @@ bool decode_deflate(int fd,
     (void)out_device;
     struct libdeflate_decompressor* d;
 
+    if (dest == nullptr)
+    {
+        throw std::runtime_error("'dest' shouldn't be nullptr in decode_deflate()");
+    }
+
+    // Allocate memory only when dest is not null
+    if (*dest == nullptr)
+    {
+        if ((*dest = (unsigned char*)malloc(dest_nbytes)) == nullptr)
+        {
+            throw std::runtime_error("Unable to allocate uncompressed image buffer");
+        }
+    }
+
     d = libdeflate_alloc_decompressor();
 
     if (d == nullptr)

cpp/plugins/cucim.kit.cuslide/src/cuslide/jpeg/libjpeg_turbo.cpp

@@ -108,6 +108,11 @@ bool decode_libjpeg(int fd,
     if (size == 0)
         THROW("determining input file size", "Input file contains no data");
 
+    if (dest == nullptr)
+    {
+        THROW("checking dest ptr", "'dest' shouldn't be nullptr in decode_libjpeg()");
+    }
+
     if (jpeg_buf == nullptr)
     {
         if ((jpeg_buf = (unsigned char*)tjAlloc(size)) == nullptr)
@@ -145,7 +150,7 @@ bool decode_libjpeg(int fd,
     if (*dest == nullptr)
     {
         if ((*dest = (unsigned char*)tjAlloc(width * height * tjPixelSize[pixelFormat])) == nullptr)
-            THROW_UNIX("allocating uncompressed image buffer");
+            THROW_UNIX("Unable to allocate uncompressed image buffer");
     }
 
     if (tjDecompress2(tjInstance, jpeg_buf, size, (unsigned char*)*dest, width, 0, height, pixelFormat, flags) < 0)

cpp/plugins/cucim.kit.cuslide/src/cuslide/raw/raw.cpp

@@ -0,0 +1,85 @@
+/*
+ * Apache License, Version 2.0
+ * Copyright 2021 NVIDIA Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Code below is using libdeflate library which is under MIT license
+ * Please see LICENSE-3rdparty.md for the detail.
+ */
+
+#include "raw.h"
+
+#include <cstring>
+#include <stdexcept>
+#include <unistd.h>
+
+
+namespace cuslide::raw
+{
+
+bool decode_raw(int fd,
+                unsigned char* raw_buf,
+                uint64_t offset,
+                uint64_t size,
+                uint8_t** dest,
+                uint64_t dest_nbytes,
+                const cucim::io::Device& out_device)
+{
+    (void)out_device;
+
+    if (dest == nullptr)
+    {
+        throw std::runtime_error("'dest' shouldn't be nullptr in decode_raw()");
+    }
+
+    // Allocate memory only when dest is not null
+    if (*dest == nullptr)
+    {
+        if ((*dest = (unsigned char*)malloc(dest_nbytes)) == nullptr)
+        {
+            throw std::runtime_error("Unable to allocate uncompressed image buffer");
+        }
+    }
+
+    if (raw_buf == nullptr)
+    {
+        if ((raw_buf = (unsigned char*)malloc(size)) == nullptr)
+        {
+            throw std::runtime_error("Unable to allocate buffer for raw data!");
+        }
+
+        if (pread(fd, raw_buf, size, offset) < 1)
+        {
+            throw std::runtime_error("Unable to read file for raw data!");
+        }
+    }
+    else
+    {
+        fd = -1;
+        raw_buf += offset;
+    }
+
+    memcpy(*dest, raw_buf, dest_nbytes);
+
+    if (fd != -1)
+    {
+        free(raw_buf);
+    }
+
+    return true;
+}
+
+} // namespace cuslide::raw

cpp/plugins/cucim.kit.cuslide/src/cuslide/raw/raw.h

@@ -0,0 +1,33 @@
+/*
+ * Apache License, Version 2.0
+ * Copyright 2021 NVIDIA Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef CUSLIDE_RAW_H
+#define CUSLIDE_RAW_H
+
+#include <cucim/io/device.h>
+
+namespace cuslide::raw
+{
+
+bool decode_raw(int fd,
+                unsigned char* raw_buf,
+                uint64_t offset,
+                uint64_t size,
+                uint8_t** dest,
+                uint64_t dest_nbytes,
+                const cucim::io::Device& out_device);
+}
+#endif // CUSLIDE_RAW_H

cpp/plugins/cucim.kit.cuslide/src/cuslide/tiff/ifd.cpp

@@ -34,6 +34,7 @@
 
 #include "cuslide/jpeg/libjpeg_turbo.h"
 #include "cuslide/deflate/deflate.h"
+#include "cuslide/raw/raw.h"
 #include "tiff.h"
 
 
@@ -151,7 +152,7 @@ bool IFD::read(const TIFF* tiff,
     int64_t sy = request->location[1];
     int64_t w = request->size[0];
     int64_t h = request->size[1];
-    int32_t n_ch = 3; // number of channels
+    int32_t n_ch = samples_per_pixel_; // number of channels
 
     size_t raster_size = w * h * samples_per_pixel_;
     void* raster = nullptr;
@@ -365,13 +366,22 @@ const std::vector<uint64_t>& IFD::image_piece_bytecounts() const
 
 bool IFD::is_compression_supported() const
 {
-    return (compression_ == COMPRESSION_ADOBE_DEFLATE || compression_ == COMPRESSION_JPEG ||
-            compression_ == COMPRESSION_DEFLATE);
+    switch (compression_)
+    {
+    case COMPRESSION_NONE:
+    case COMPRESSION_JPEG:
+    case COMPRESSION_ADOBE_DEFLATE:
+    case COMPRESSION_DEFLATE:
+        return true;
+    default:
+        return false;
+    }
 }
+
 bool IFD::is_read_optimizable() const
 {
     return is_compression_supported() && bits_per_sample_ == 8 && samples_per_pixel_ == 3 &&
-           planar_config_ == PLANARCONFIG_CONTIG &&
+           (tile_width_ != 0 && tile_height_ != 0) && planar_config_ == PLANARCONFIG_CONTIG &&
            (photometric_ == PHOTOMETRIC_RGB || photometric_ == PHOTOMETRIC_YCBCR) &&
            !tiff_->is_in_read_config(TIFF::kUseLibTiff);
 }
@@ -501,7 +511,12 @@ bool IFD::read_region_tiles(const TIFF* tiff,
                         tile_data = static_cast<uint8_t*>(image_cache.allocate(tile_raster_nbytes));
                     }
 
-                    if (compression_method == COMPRESSION_JPEG)
+                    if (compression_method == COMPRESSION_NONE)
+                    {
+                        cuslide::raw::decode_raw(tiff_file, nullptr, tiledata_offset, tiledata_size, &tile_data,
+                                                 tile_raster_nbytes, out_device);
+                    }
+                    else if (compression_method == COMPRESSION_JPEG)
                     {
                         cuslide::jpeg::decode_libjpeg(tiff_file, nullptr, tiledata_offset, tiledata_size,
                                                       jpegtable_data, jpegtable_count, &tile_data, out_device);
@@ -745,7 +760,13 @@ bool IFD::read_region_tiles_boundary(const TIFF* tiff,
                     {
                         tile_data = static_cast<uint8_t*>(image_cache.allocate(tile_raster_nbytes));
                     }
-                    if (compression_method == COMPRESSION_JPEG)
+
+                    if (compression_method == COMPRESSION_NONE)
+                    {
+                        cuslide::raw::decode_raw(tiff_file, nullptr, tiledata_offset, tiledata_size, &tile_data,
+                                                 tile_raster_nbytes, out_device);
+                    }
+                    else if (compression_method == COMPRESSION_JPEG)
                     {
                         cuslide::jpeg::decode_libjpeg(tiff_file, nullptr, tiledata_offset, tiledata_size,
                                                       jpegtable_data, jpegtable_count, &tile_data, out_device);