Unmanaged pooling MemoryAllocator

.gitattributes

@@ -87,15 +87,13 @@
 *.eot               binary
 *.exe               binary
 *.otf               binary
-*.pbm               binary
 *.pdf               binary
 *.ppt               binary
 *.pptx              binary
 *.pvr               binary
 *.snk               binary
 *.ttc               binary
 *.ttf               binary
-*.wbmp              binary
 *.woff              binary
 *.woff2             binary
 *.xls               binary
@@ -126,3 +124,9 @@
 *.dds               filter=lfs diff=lfs merge=lfs -text
 *.ktx               filter=lfs diff=lfs merge=lfs -text
 *.ktx2              filter=lfs diff=lfs merge=lfs -text
+*.pam               filter=lfs diff=lfs merge=lfs -text
+*.pbm               filter=lfs diff=lfs merge=lfs -text
+*.pgm               filter=lfs diff=lfs merge=lfs -text
+*.ppm               filter=lfs diff=lfs merge=lfs -text
+*.pnm               filter=lfs diff=lfs merge=lfs -text
+*.wbmp              filter=lfs diff=lfs merge=lfs -text

Directory.Build.props

@@ -13,6 +13,9 @@
   <PropertyGroup>
     <!-- This MUST be defined before importing props. -->
     <SixLaborsSolutionDirectory>$(MSBuildThisFileDirectory)</SixLaborsSolutionDirectory>
+
+    <!-- For some reason Debug-InnerLoop doesn't define DEBUG by default. -->
+    <DefineConstants Condition="'$(Configuration)' == 'Debug-InnerLoop'">$(DefineConstants);DEBUG</DefineConstants>
   </PropertyGroup>
 
   <!-- Import the shared global .props file -->
@@ -30,5 +33,4 @@
   <PropertyGroup Condition="$(Configuration.StartsWith('Release')) == true">
     <Optimize>true</Optimize>
   </PropertyGroup>
-
 </Project>

src/ImageSharp/Advanced/AdvancedImageExtensions.cs

@@ -143,7 +143,7 @@ public static IMemoryGroup<TPixel> GetPixelMemoryGroup<TPixel>(this Image<TPixel
         /// <param name="source">The source.</param>
         /// <param name="rowIndex">The row.</param>
         /// <returns>The <see cref="Span{TPixel}"/></returns>
-        public static Memory<TPixel> GetPixelRowMemory<TPixel>(this ImageFrame<TPixel> source, int rowIndex)
+        public static Memory<TPixel> DangerousGetPixelRowMemory<TPixel>(this ImageFrame<TPixel> source, int rowIndex)
             where TPixel : unmanaged, IPixel<TPixel>
         {
             Guard.NotNull(source, nameof(source));
@@ -161,7 +161,7 @@ public static Memory<TPixel> GetPixelRowMemory<TPixel>(this ImageFrame<TPixel> s
         /// <param name="source">The source.</param>
         /// <param name="rowIndex">The row.</param>
         /// <returns>The <see cref="Span{TPixel}"/></returns>
-        public static Memory<TPixel> GetPixelRowMemory<TPixel>(this Image<TPixel> source, int rowIndex)
+        public static Memory<TPixel> DangerousGetPixelRowMemory<TPixel>(this Image<TPixel> source, int rowIndex)
             where TPixel : unmanaged, IPixel<TPixel>
         {
             Guard.NotNull(source, nameof(source));

src/ImageSharp/Advanced/AotCompilerTools.cs

@@ -529,7 +529,7 @@ private static void AotCompileDither<TPixel, TDither>()
         private static void AotCompileMemoryManagers<TPixel>()
             where TPixel : unmanaged, IPixel<TPixel>
         {
-            AotCompileMemoryManager<TPixel, ArrayPoolMemoryAllocator>();
+            AotCompileMemoryManager<TPixel, UniformUnmanagedMemoryPoolMemoryAllocator>();
             AotCompileMemoryManager<TPixel, SimpleGcMemoryAllocator>();
         }
 

src/ImageSharp/Common/Helpers/DebugGuard.cs

@@ -26,6 +26,20 @@ public static void IsTrue(bool target, string message)
             }
         }
 
+        /// <summary>
+        /// Verifies whether a condition (indicating disposed state) is met, throwing an ObjectDisposedException if it's true.
+        /// </summary>
+        /// <param name="isDisposed">Whether the object is disposed.</param>
+        /// <param name="objectName">The name of the object.</param>
+        [Conditional("DEBUG")]
+        public static void NotDisposed(bool isDisposed, string objectName)
+        {
+            if (isDisposed)
+            {
+                throw new ObjectDisposedException(objectName);
+            }
+        }
+
         /// <summary>
         /// Verifies, that the target span is of same size than the 'other' span.
         /// </summary>

src/ImageSharp/Common/Helpers/Shuffle/IComponentShuffle.cs

@@ -28,6 +28,10 @@ internal interface IComponentShuffle
         /// </summary>
         /// <param name="source">The source span of bytes.</param>
         /// <param name="dest">The destination span of bytes.</param>
+        /// <remarks>
+        /// Implementation can assume that source.Length is less or equal than dest.Length.
+        /// Loops should iterate using source.Length.
+        /// </remarks>
         void RunFallbackShuffle(ReadOnlySpan<byte> source, Span<byte> dest);
     }
 

src/ImageSharp/Common/Helpers/SimdUtils.Shuffle.cs

@@ -77,6 +77,7 @@ public static void Shuffle3<TShuffle>(
             TShuffle shuffle)
             where TShuffle : struct, IShuffle3
         {
+            // Source length should be smaller than dest length, and divisible by 3.
             VerifyShuffle3SpanInput(source, dest);
 
 #if SUPPORTS_RUNTIME_INTRINSICS
@@ -182,9 +183,9 @@ private static void VerifyShuffle3SpanInput<T>(ReadOnlySpan<T> source, Span<T> d
             where T : struct
         {
             DebugGuard.IsTrue(
-                source.Length == dest.Length,
+                source.Length <= dest.Length,
                 nameof(source),
-                "Input spans must be of same length!");
+                "Source should fit into dest!");
 
             DebugGuard.IsTrue(
                 source.Length % 3 == 0,

src/ImageSharp/Configuration.cs

@@ -26,10 +26,11 @@ public sealed class Configuration
         /// <summary>
         /// A lazily initialized configuration default instance.
         /// </summary>
-        private static readonly Lazy<Configuration> Lazy = new Lazy<Configuration>(CreateDefaultInstance);
+        private static readonly Lazy<Configuration> Lazy = new(CreateDefaultInstance);
         private const int DefaultStreamProcessingBufferSize = 8096;
         private int streamProcessingBufferSize = DefaultStreamProcessingBufferSize;
         private int maxDegreeOfParallelism = Environment.ProcessorCount;
+        private MemoryAllocator memoryAllocator = MemoryAllocator.Default;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="Configuration" /> class.
@@ -95,6 +96,14 @@ public int StreamProcessingBufferSize
             }
         }
 
+        /// <summary>
+        /// Gets or sets a value indicating whether to force image buffers to be contiguous whenever possible.
+        /// </summary>
+        /// <remarks>
+        /// Contiguous allocations are not possible, if the image needs a buffer larger than <see cref="int.MaxValue"/>.
+        /// </remarks>
+        public bool PreferContiguousImageBuffers { get; set; }
+
         /// <summary>
         /// Gets a set of properties for the Configuration.
         /// </summary>
@@ -117,9 +126,31 @@ public int StreamProcessingBufferSize
         public ImageFormatManager ImageFormatsManager { get; set; } = new ImageFormatManager();
 
         /// <summary>
-        /// Gets or sets the <see cref="MemoryAllocator"/> that is currently in use.
+        /// Gets or sets the <see cref="ImageSharp.Memory.MemoryAllocator"/> that is currently in use.
+        /// Defaults to <see cref="ImageSharp.Memory.MemoryAllocator.Default"/>.
+        /// <para />
+        /// Allocators are expensive, so it is strongly recommended to use only one busy instance per process.
+        /// In case you need to customize it, you can ensure this by changing
         /// </summary>
-        public MemoryAllocator MemoryAllocator { get; set; } = ArrayPoolMemoryAllocator.CreateDefault();
+        /// <remarks>
+        /// It's possible to reduce allocator footprint by assigning a custom instance created with
+        /// <see cref="Memory.MemoryAllocator.Create(MemoryAllocatorOptions)"/>, but note that since the default pooling
+        /// allocators are expensive, it is strictly recommended to use a single process-wide allocator.
+        /// You can ensure this by altering the allocator of <see cref="Default"/>, or by implementing custom application logic that
+        /// manages allocator lifetime.
+        /// <para />
+        /// If an allocator has to be dropped for some reason, <see cref="Memory.MemoryAllocator.ReleaseRetainedResources"/>
+        /// shall be invoked after disposing all associated <see cref="Image"/> instances.
+        /// </remarks>
+        public MemoryAllocator MemoryAllocator
+        {
+            get => this.memoryAllocator;
+            set
+            {
+                Guard.NotNull(value, nameof(this.MemoryAllocator));
+                this.memoryAllocator = value;
+            }
+        }
 
         /// <summary>
         /// Gets the maximum header size of all the formats.
@@ -165,7 +196,7 @@ public void Configure(IConfigurationModule configuration)
             MaxDegreeOfParallelism = this.MaxDegreeOfParallelism,
             StreamProcessingBufferSize = this.StreamProcessingBufferSize,
             ImageFormatsManager = this.ImageFormatsManager,
-            MemoryAllocator = this.MemoryAllocator,
+            memoryAllocator = this.memoryAllocator,
             ImageOperationsProvider = this.ImageOperationsProvider,
             ReadOrigin = this.ReadOrigin,
             FileSystem = this.FileSystem,

src/ImageSharp/Formats/Bmp/BmpDecoderCore.cs

@@ -306,7 +306,7 @@ private void ReadRle<TPixel>(BmpCompression compression, Buffer2D<TPixel> pixels
                     int newY = Invert(y, height, inverted);
                     int rowStartIdx = y * width;
                     Span<byte> bufferRow = bufferSpan.Slice(rowStartIdx, width);
-                    Span<TPixel> pixelRow = pixels.GetRowSpan(newY);
+                    Span<TPixel> pixelRow = pixels.DangerousGetRowSpan(newY);
 
                     bool rowHasUndefinedPixels = rowsWithUndefinedPixelsSpan[y];
                     if (rowHasUndefinedPixels)
@@ -377,7 +377,7 @@ private void ReadRle24<TPixel>(Buffer2D<TPixel> pixels, int width, int height, b
                 for (int y = 0; y < height; y++)
                 {
                     int newY = Invert(y, height, inverted);
-                    Span<TPixel> pixelRow = pixels.GetRowSpan(newY);
+                    Span<TPixel> pixelRow = pixels.DangerousGetRowSpan(newY);
                     bool rowHasUndefinedPixels = rowsWithUndefinedPixelsSpan[y];
                     if (rowHasUndefinedPixels)
                     {
@@ -826,7 +826,7 @@ private void ReadRgbPalette<TPixel>(Buffer2D<TPixel> pixels, byte[] colors, int
                 int newY = Invert(y, height, inverted);
                 this.stream.Read(rowSpan);
                 int offset = 0;
-                Span<TPixel> pixelRow = pixels.GetRowSpan(newY);
+                Span<TPixel> pixelRow = pixels.DangerousGetRowSpan(newY);
 
                 for (int x = 0; x < arrayWidth; x++)
                 {
@@ -878,7 +878,7 @@ private void ReadRgb16<TPixel>(Buffer2D<TPixel> pixels, int width, int height, b
             {
                 this.stream.Read(bufferSpan);
                 int newY = Invert(y, height, inverted);
-                Span<TPixel> pixelRow = pixels.GetRowSpan(newY);
+                Span<TPixel> pixelRow = pixels.DangerousGetRowSpan(newY);
 
                 int offset = 0;
                 for (int x = 0; x < width; x++)
@@ -933,7 +933,7 @@ private void ReadRgb24<TPixel>(Buffer2D<TPixel> pixels, int width, int height, b
             {
                 this.stream.Read(rowSpan);
                 int newY = Invert(y, height, inverted);
-                Span<TPixel> pixelSpan = pixels.GetRowSpan(newY);
+                Span<TPixel> pixelSpan = pixels.DangerousGetRowSpan(newY);
                 PixelOperations<TPixel>.Instance.FromBgr24Bytes(
                     this.Configuration,
                     rowSpan,
@@ -961,7 +961,7 @@ private void ReadRgb32Fast<TPixel>(Buffer2D<TPixel> pixels, int width, int heigh
             {
                 this.stream.Read(rowSpan);
                 int newY = Invert(y, height, inverted);
-                Span<TPixel> pixelSpan = pixels.GetRowSpan(newY);
+                Span<TPixel> pixelSpan = pixels.DangerousGetRowSpan(newY);
                 PixelOperations<TPixel>.Instance.FromBgra32Bytes(
                     this.Configuration,
                     rowSpan,
@@ -1031,7 +1031,7 @@ private void ReadRgb32Slow<TPixel>(Buffer2D<TPixel> pixels, int width, int heigh
                     this.stream.Read(rowSpan);
 
                     int newY = Invert(y, height, inverted);
-                    Span<TPixel> pixelSpan = pixels.GetRowSpan(newY);
+                    Span<TPixel> pixelSpan = pixels.DangerousGetRowSpan(newY);
 
                     PixelOperations<TPixel>.Instance.FromBgra32Bytes(
                         this.Configuration,
@@ -1054,7 +1054,7 @@ private void ReadRgb32Slow<TPixel>(Buffer2D<TPixel> pixels, int width, int heigh
                     width);
 
                 int newY = Invert(y, height, inverted);
-                Span<TPixel> pixelSpan = pixels.GetRowSpan(newY);
+                Span<TPixel> pixelSpan = pixels.DangerousGetRowSpan(newY);
 
                 for (int x = 0; x < width; x++)
                 {
@@ -1109,7 +1109,7 @@ private void ReadRgb32BitFields<TPixel>(Buffer2D<TPixel> pixels, int width, int
             {
                 this.stream.Read(bufferSpan);
                 int newY = Invert(y, height, inverted);
-                Span<TPixel> pixelRow = pixels.GetRowSpan(newY);
+                Span<TPixel> pixelRow = pixels.DangerousGetRowSpan(newY);
 
                 int offset = 0;
                 for (int x = 0; x < width; x++)

src/ImageSharp/Formats/Bmp/BmpEncoderCore.cs

@@ -274,7 +274,7 @@ private void Write32Bit<TPixel>(Stream stream, Buffer2D<TPixel> pixels)
 
             for (int y = pixels.Height - 1; y >= 0; y--)
             {
-                Span<TPixel> pixelSpan = pixels.GetRowSpan(y);
+                Span<TPixel> pixelSpan = pixels.DangerousGetRowSpan(y);
                 PixelOperations<TPixel>.Instance.ToBgra32Bytes(
                     this.configuration,
                     pixelSpan,
@@ -300,7 +300,7 @@ private void Write24Bit<TPixel>(Stream stream, Buffer2D<TPixel> pixels)
 
             for (int y = pixels.Height - 1; y >= 0; y--)
             {
-                Span<TPixel> pixelSpan = pixels.GetRowSpan(y);
+                Span<TPixel> pixelSpan = pixels.DangerousGetRowSpan(y);
                 PixelOperations<TPixel>.Instance.ToBgr24Bytes(
                     this.configuration,
                     pixelSpan,
@@ -326,7 +326,7 @@ private void Write16Bit<TPixel>(Stream stream, Buffer2D<TPixel> pixels)
 
             for (int y = pixels.Height - 1; y >= 0; y--)
             {
-                Span<TPixel> pixelSpan = pixels.GetRowSpan(y);
+                Span<TPixel> pixelSpan = pixels.DangerousGetRowSpan(y);
 
                 PixelOperations<TPixel>.Instance.ToBgra5551Bytes(
                     this.configuration,
@@ -379,7 +379,7 @@ private void Write8BitColor<TPixel>(Stream stream, ImageFrame<TPixel> image, Spa
 
             for (int y = image.Height - 1; y >= 0; y--)
             {
-                ReadOnlySpan<byte> pixelSpan = quantized.GetPixelRowSpan(y);
+                ReadOnlySpan<byte> pixelSpan = quantized.DangerousGetRowSpan(y);
                 stream.Write(pixelSpan);
 
                 for (int i = 0; i < this.padding; i++)
@@ -413,10 +413,10 @@ private void Write8BitGray<TPixel>(Stream stream, ImageFrame<TPixel> image, Span
             }
 
             stream.Write(colorPalette);
-
+            Buffer2D<TPixel> imageBuffer = image.PixelBuffer;
             for (int y = image.Height - 1; y >= 0; y--)
             {
-                ReadOnlySpan<TPixel> inputPixelRow = image.GetPixelRowSpan(y);
+                ReadOnlySpan<TPixel> inputPixelRow = imageBuffer.DangerousGetRowSpan(y);
                 ReadOnlySpan<byte> outputPixelRow = MemoryMarshal.AsBytes(inputPixelRow);
                 stream.Write(outputPixelRow);
 
@@ -447,11 +447,11 @@ private void Write4BitColor<TPixel>(Stream stream, ImageFrame<TPixel> image)
             ReadOnlySpan<TPixel> quantizedColorPalette = quantized.Palette.Span;
             this.WriteColorPalette(stream, quantizedColorPalette, colorPalette);
 
-            ReadOnlySpan<byte> pixelRowSpan = quantized.GetPixelRowSpan(0);
+            ReadOnlySpan<byte> pixelRowSpan = quantized.DangerousGetRowSpan(0);
             int rowPadding = pixelRowSpan.Length % 2 != 0 ? this.padding - 1 : this.padding;
             for (int y = image.Height - 1; y >= 0; y--)
             {
-                pixelRowSpan = quantized.GetPixelRowSpan(y);
+                pixelRowSpan = quantized.DangerousGetRowSpan(y);
 
                 int endIdx = pixelRowSpan.Length % 2 == 0 ? pixelRowSpan.Length : pixelRowSpan.Length - 1;
                 for (int i = 0; i < endIdx; i += 2)
@@ -491,11 +491,11 @@ private void Write1BitColor<TPixel>(Stream stream, ImageFrame<TPixel> image)
             ReadOnlySpan<TPixel> quantizedColorPalette = quantized.Palette.Span;
             this.WriteColorPalette(stream, quantizedColorPalette, colorPalette);
 
-            ReadOnlySpan<byte> quantizedPixelRow = quantized.GetPixelRowSpan(0);
+            ReadOnlySpan<byte> quantizedPixelRow = quantized.DangerousGetRowSpan(0);
             int rowPadding = quantizedPixelRow.Length % 8 != 0 ? this.padding - 1 : this.padding;
             for (int y = image.Height - 1; y >= 0; y--)
             {
-                quantizedPixelRow = quantized.GetPixelRowSpan(y);
+                quantizedPixelRow = quantized.DangerousGetRowSpan(y);
 
                 int endIdx = quantizedPixelRow.Length % 8 == 0 ? quantizedPixelRow.Length : quantizedPixelRow.Length - 8;
                 for (int i = 0; i < endIdx; i += 8)

src/ImageSharp/Formats/Gif/GifDecoderCore.cs

@@ -445,7 +445,7 @@ private void ReadFrameColors<TPixel>(ref Image<TPixel> image, ref ImageFrame<TPi
 
             for (int y = descriptorTop; y < descriptorBottom && y < imageHeight; y++)
             {
-                ref byte indicesRowRef = ref MemoryMarshal.GetReference(indices.GetRowSpan(y - descriptorTop));
+                ref byte indicesRowRef = ref MemoryMarshal.GetReference(indices.DangerousGetRowSpan(y - descriptorTop));
 
                 // Check if this image is interlaced.
                 int writeY; // the target y offset to write to
@@ -481,7 +481,7 @@ private void ReadFrameColors<TPixel>(ref Image<TPixel> image, ref ImageFrame<TPi
                     writeY = y;
                 }
 
-                ref TPixel rowRef = ref MemoryMarshal.GetReference(imageFrame.GetPixelRowSpan(writeY));
+                ref TPixel rowRef = ref MemoryMarshal.GetReference(imageFrame.PixelBuffer.DangerousGetRowSpan(writeY));
 
                 if (!transFlag)
                 {

src/ImageSharp/Formats/Gif/LzwDecoder.cs

@@ -115,14 +115,14 @@ public void DecodePixels(int dataSize, Buffer2D<byte> pixels)
             int y = 0;
             int x = 0;
             int rowMax = width;
-            ref byte pixelsRowRef = ref MemoryMarshal.GetReference(pixels.GetRowSpan(y));
+            ref byte pixelsRowRef = ref MemoryMarshal.GetReference(pixels.DangerousGetRowSpan(y));
             while (xyz < length)
             {
                 // Reset row reference.
                 if (xyz == rowMax)
                 {
                     x = 0;
-                    pixelsRowRef = ref MemoryMarshal.GetReference(pixels.GetRowSpan(++y));
+                    pixelsRowRef = ref MemoryMarshal.GetReference(pixels.DangerousGetRowSpan(++y));
                     rowMax = (y * width) + width;
                 }
 

src/ImageSharp/Formats/Gif/LzwEncoder.cs

@@ -275,7 +275,7 @@ private void Compress(Buffer2D<byte> indexedPixels, int initialBits, Stream stre
 
             for (int y = 0; y < indexedPixels.Height; y++)
             {
-                ref byte rowSpanRef = ref MemoryMarshal.GetReference(indexedPixels.GetRowSpan(y));
+                ref byte rowSpanRef = ref MemoryMarshal.GetReference(indexedPixels.DangerousGetRowSpan(y));
                 int offsetX = y == 0 ? 1 : 0;
 
                 for (int x = offsetX; x < indexedPixels.Width; x++)