diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs index 9de591aaaa961..3a316d8f74089 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Buffers.Binary; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Numerics; @@ -1130,23 +1129,21 @@ private static unsafe nuint UnalignedCountVector128(ref byte searchSpace) public static void Reverse(ref byte buf, nuint length) { - Debug.Assert(length > 1); - - nint remainder = (nint)length; - nint offset = 0; - - if (Avx2.IsSupported && remainder >= Vector256.Count) + if (Avx2.IsSupported && (nuint)Vector256.Count * 2 <= length) { Vector256 reverseMask = Vector256.Create( (byte)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, // first 128-bit lane 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); // second 128-bit lane - - nint lastOffset = remainder - Vector256.Count; - do + nuint numElements = (nuint)Vector256.Count; + nuint numIters = (length / numElements) / 2; + for (nuint i = 0; i < numIters; i++) { - // Load the values into vectors - Vector256 tempFirst = Vector256.LoadUnsafe(ref buf, (nuint)offset); - Vector256 tempLast = Vector256.LoadUnsafe(ref buf, (nuint)lastOffset); + nuint firstOffset = i * numElements; + nuint lastOffset = length - ((1 + i) * numElements); + + // Load in values from beginning and end of the array. + Vector256 tempFirst = Vector256.LoadUnsafe(ref buf, firstOffset); + Vector256 tempLast = Vector256.LoadUnsafe(ref buf, lastOffset); // Avx2 operates on two 128-bit lanes rather than the full 256-bit vector. // Perform a shuffle to reverse each 128-bit lane, then permute to finish reversing the vector: @@ -1173,23 +1170,24 @@ public static void Reverse(ref byte buf, nuint length) tempLast = Avx2.Permute2x128(tempLast, tempLast, 0b00_01); // Store the reversed vectors - tempLast.StoreUnsafe(ref buf, (nuint)offset); - tempFirst.StoreUnsafe(ref buf, (nuint)lastOffset); - - offset += Vector256.Count; - lastOffset -= Vector256.Count; - } while (lastOffset >= offset); - - remainder = lastOffset + Vector256.Count - offset; + tempLast.StoreUnsafe(ref buf, firstOffset); + tempFirst.StoreUnsafe(ref buf, lastOffset); + } + buf = ref Unsafe.Add(ref buf, numIters * numElements); + length -= numIters * numElements * 2; } - else if (Vector128.IsHardwareAccelerated && remainder >= Vector128.Count) + else if (Vector128.IsHardwareAccelerated && (nuint)Vector128.Count * 2 <= length) { - nint lastOffset = remainder - Vector128.Count; - do + nuint numElements = (nuint)Vector128.Count; + nuint numIters = (length / numElements) / 2; + for (nuint i = 0; i < numIters; i++) { - // Load the values into vectors - Vector128 tempFirst = Vector128.LoadUnsafe(ref buf, (nuint)offset); - Vector128 tempLast = Vector128.LoadUnsafe(ref buf, (nuint)lastOffset); + nuint firstOffset = i * numElements; + nuint lastOffset = length - ((1 + i) * numElements); + + // Load in values from beginning and end of the array. + Vector128 tempFirst = Vector128.LoadUnsafe(ref buf, firstOffset); + Vector128 tempLast = Vector128.LoadUnsafe(ref buf, lastOffset); // Shuffle to reverse each vector: // +---------------------------------------------------------------+ @@ -1205,58 +1203,15 @@ public static void Reverse(ref byte buf, nuint length) (byte)15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)); // Store the reversed vectors - tempLast.StoreUnsafe(ref buf, (nuint)offset); - tempFirst.StoreUnsafe(ref buf, (nuint)lastOffset); - - offset += Vector128.Count; - lastOffset -= Vector128.Count; - } while (lastOffset >= offset); - - remainder = lastOffset + Vector128.Count - offset; - } - - if (remainder >= sizeof(long)) - { - nint lastOffset = (nint)length - offset - sizeof(long); - do - { - long tempFirst = Unsafe.ReadUnaligned(ref Unsafe.Add(ref buf, offset)); - long tempLast = Unsafe.ReadUnaligned(ref Unsafe.Add(ref buf, lastOffset)); - - // swap and store in reversed position - Unsafe.WriteUnaligned(ref Unsafe.Add(ref buf, offset), BinaryPrimitives.ReverseEndianness(tempLast)); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref buf, lastOffset), BinaryPrimitives.ReverseEndianness(tempFirst)); - - offset += sizeof(long); - lastOffset -= sizeof(long); - } while (lastOffset >= offset); - - remainder = lastOffset + sizeof(long) - offset; - } - - if (remainder >= sizeof(int)) - { - nint lastOffset = (nint)length - offset - sizeof(int); - do - { - int tempFirst = Unsafe.ReadUnaligned(ref Unsafe.Add(ref buf, offset)); - int tempLast = Unsafe.ReadUnaligned(ref Unsafe.Add(ref buf, lastOffset)); - - // swap and store in reversed position - Unsafe.WriteUnaligned(ref Unsafe.Add(ref buf, offset), BinaryPrimitives.ReverseEndianness(tempLast)); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref buf, lastOffset), BinaryPrimitives.ReverseEndianness(tempFirst)); - - offset += sizeof(int); - lastOffset -= sizeof(int); - } while (lastOffset >= offset); - - remainder = lastOffset + sizeof(int) - offset; + tempLast.StoreUnsafe(ref buf, firstOffset); + tempFirst.StoreUnsafe(ref buf, lastOffset); + } + buf = ref Unsafe.Add(ref buf, numIters * numElements); + length -= numIters * numElements * 2; } - if (remainder > 1) - { - ReverseInner(ref Unsafe.Add(ref buf, offset), (nuint)remainder); - } + // Store any remaining values one-by-one + ReverseInner(ref buf, length); } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs index ca1af16e719c6..8a44db8cc3aed 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs @@ -733,25 +733,23 @@ private static unsafe nint UnalignedCountVector128(ref char searchSpace) public static void Reverse(ref char buf, nuint length) { - Debug.Assert(length > 1); - - nint remainder = (nint)length; - nint offset = 0; - - if (Avx2.IsSupported && remainder >= Vector256.Count) + if (Avx2.IsSupported && (nuint)Vector256.Count * 2 <= length) { + ref byte bufByte = ref Unsafe.As(ref buf); + nuint byteLength = length * sizeof(char); Vector256 reverseMask = Vector256.Create( (byte)14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, // first 128-bit lane 14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); // second 128-bit lane - - nint lastOffset = remainder - Vector256.Count; - do + nuint numElements = (nuint)Vector256.Count; + nuint numIters = (byteLength / numElements) / 2; + for (nuint i = 0; i < numIters; i++) { - ref byte first = ref Unsafe.As(ref Unsafe.Add(ref buf, offset)); - ref byte last = ref Unsafe.As(ref Unsafe.Add(ref buf, lastOffset)); + nuint firstOffset = i * numElements; + nuint lastOffset = byteLength - ((1 + i) * numElements); - Vector256 tempFirst = Vector256.LoadUnsafe(ref first); - Vector256 tempLast = Vector256.LoadUnsafe(ref last); + // Load in values from beginning and end of the array. + Vector256 tempFirst = Vector256.LoadUnsafe(ref bufByte, firstOffset); + Vector256 tempLast = Vector256.LoadUnsafe(ref bufByte, lastOffset); // Avx2 operates on two 128-bit lanes rather than the full 256-bit vector. // Perform a shuffle to reverse each 128-bit lane, then permute to finish reversing the vector: @@ -772,25 +770,27 @@ public static void Reverse(ref char buf, nuint length) tempLast = Avx2.Permute2x128(tempLast, tempLast, 0b00_01); // Store the reversed vectors - tempLast.StoreUnsafe(ref first); - tempFirst.StoreUnsafe(ref last); - - offset += Vector256.Count; - lastOffset -= Vector256.Count; - } while (lastOffset >= offset); - - remainder = (lastOffset + Vector256.Count - offset); + tempLast.StoreUnsafe(ref bufByte, firstOffset); + tempFirst.StoreUnsafe(ref bufByte, lastOffset); + } + bufByte = ref Unsafe.Add(ref bufByte, numIters * numElements); + length -= numIters * (nuint)Vector256.Count * 2; + // Store any remaining values one-by-one + buf = ref Unsafe.As(ref bufByte); } - else if (Vector128.IsHardwareAccelerated && remainder >= Vector128.Count) + else if (Vector128.IsHardwareAccelerated && (nuint)Vector128.Count * 2 <= length) { - nint lastOffset = remainder - Vector128.Count; - do + ref short bufShort = ref Unsafe.As(ref buf); + nuint numElements = (nuint)Vector128.Count; + nuint numIters = (length / numElements) / 2; + for (nuint i = 0; i < numIters; i++) { - ref ushort first = ref Unsafe.As(ref Unsafe.Add(ref buf, offset)); - ref ushort last = ref Unsafe.As(ref Unsafe.Add(ref buf, lastOffset)); + nuint firstOffset = i * numElements; + nuint lastOffset = length - ((1 + i) * numElements); - Vector128 tempFirst = Vector128.LoadUnsafe(ref first); - Vector128 tempLast = Vector128.LoadUnsafe(ref last); + // Load in values from beginning and end of the array. + Vector128 tempFirst = Vector128.LoadUnsafe(ref bufShort, firstOffset); + Vector128 tempLast = Vector128.LoadUnsafe(ref bufShort, lastOffset); // Shuffle to reverse each vector: // +-------------------------------+ @@ -800,25 +800,19 @@ public static void Reverse(ref char buf, nuint length) // +-------------------------------+ // | H | G | F | E | D | C | B | A | // +-------------------------------+ - tempFirst = Vector128.Shuffle(tempFirst, Vector128.Create((ushort)7, 6, 5, 4, 3, 2, 1, 0)); - tempLast = Vector128.Shuffle(tempLast, Vector128.Create((ushort)7, 6, 5, 4, 3, 2, 1, 0)); + tempFirst = Vector128.Shuffle(tempFirst, Vector128.Create(7, 6, 5, 4, 3, 2, 1, 0)); + tempLast = Vector128.Shuffle(tempLast, Vector128.Create(7, 6, 5, 4, 3, 2, 1, 0)); // Store the reversed vectors - tempLast.StoreUnsafe(ref first); - tempFirst.StoreUnsafe(ref last); - - offset += Vector128.Count; - lastOffset -= Vector128.Count; - } while (lastOffset >= offset); - - remainder = (lastOffset + Vector128.Count - offset); - } - - // Store any remaining values one-by-one - if (remainder > 1) - { - ReverseInner(ref Unsafe.Add(ref buf, offset), (nuint)remainder); + tempLast.StoreUnsafe(ref bufShort, firstOffset); + tempFirst.StoreUnsafe(ref bufShort, lastOffset); + } + bufShort = ref Unsafe.Add(ref bufShort, numIters * numElements); + length -= numIters * (nuint)Vector128.Count * 2; + // Store any remaining values one-by-one + buf = ref Unsafe.As(ref bufShort); } + ReverseInner(ref buf, length); } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs index dbc9b6e79b735..fa40a769f92e3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs @@ -408,19 +408,19 @@ public static unsafe void ClearWithReferences(ref IntPtr ip, nuint pointerSizeLe public static void Reverse(ref int buf, nuint length) { - Debug.Assert(length > 1); - - nint remainder = (nint)length; - nint offset = 0; - - if (Avx2.IsSupported && remainder >= Vector256.Count) + if (Avx2.IsSupported && (nuint)Vector256.Count * 2 <= length) { - nint lastOffset = remainder - Vector256.Count; - do + nuint numElements = (nuint)Vector256.Count; + nuint numIters = (length / numElements) / 2; + Vector256 reverseMask = Vector256.Create(7, 6, 5, 4, 3, 2, 1, 0); + for (nuint i = 0; i < numIters; i++) { + nuint firstOffset = i * numElements; + nuint lastOffset = length - ((1 + i) * numElements); + // Load the values into vectors - Vector256 tempFirst = Vector256.LoadUnsafe(ref buf, (nuint)offset); - Vector256 tempLast = Vector256.LoadUnsafe(ref buf, (nuint)lastOffset); + Vector256 tempFirst = Vector256.LoadUnsafe(ref buf, firstOffset); + Vector256 tempLast = Vector256.LoadUnsafe(ref buf, lastOffset); // Permute to reverse each vector: // +-------------------------------+ @@ -430,27 +430,28 @@ public static void Reverse(ref int buf, nuint length) // +-------------------------------+ // | H | G | F | E | D | C | B | A | // +-------------------------------+ - tempFirst = Avx2.PermuteVar8x32(tempFirst, Vector256.Create(7, 6, 5, 4, 3, 2, 1, 0)); - tempLast = Avx2.PermuteVar8x32(tempLast, Vector256.Create(7, 6, 5, 4, 3, 2, 1, 0)); - - // Store the reversed vectors - tempLast.StoreUnsafe(ref buf, (nuint)offset); - tempFirst.StoreUnsafe(ref buf, (nuint)lastOffset); + tempFirst = Avx2.PermuteVar8x32(tempFirst, reverseMask); + tempLast = Avx2.PermuteVar8x32(tempLast, reverseMask); - offset += Vector256.Count; - lastOffset -= Vector256.Count; - } while (lastOffset >= offset); - - remainder = lastOffset + Vector256.Count - offset; + // Store the values into final location + tempLast.StoreUnsafe(ref buf, firstOffset); + tempFirst.StoreUnsafe(ref buf, lastOffset); + } + buf = ref Unsafe.Add(ref buf, numIters * numElements); + length -= numIters * numElements * 2; } - else if (Vector128.IsHardwareAccelerated && remainder >= Vector128.Count) + else if (Vector128.IsHardwareAccelerated && (nuint)Vector128.Count * 2 <= length) { - nint lastOffset = remainder - Vector128.Count; - do + nuint numElements = (nuint)Vector128.Count; + nuint numIters = (length / numElements) / 2; + for (nuint i = 0; i < numIters; i++) { - // Load in values from beginning and end of the array. - Vector128 tempFirst = Vector128.LoadUnsafe(ref buf, (nuint)offset); - Vector128 tempLast = Vector128.LoadUnsafe(ref buf, (nuint)lastOffset); + nuint firstOffset = i * numElements; + nuint lastOffset = length - ((1 + i) * numElements); + + // Load the values into vectors + Vector128 tempFirst = Vector128.LoadUnsafe(ref buf, firstOffset); + Vector128 tempLast = Vector128.LoadUnsafe(ref buf, lastOffset); // Shuffle to reverse each vector: // +---------------+ @@ -463,39 +464,30 @@ public static void Reverse(ref int buf, nuint length) tempFirst = Vector128.Shuffle(tempFirst, Vector128.Create(3, 2, 1, 0)); tempLast = Vector128.Shuffle(tempLast, Vector128.Create(3, 2, 1, 0)); - // Store the reversed vectors - tempLast.StoreUnsafe(ref buf, (nuint)offset); - tempFirst.StoreUnsafe(ref buf, (nuint)lastOffset); - - offset += Vector128.Count; - lastOffset -= Vector128.Count; - } while (lastOffset >= offset); - - remainder = lastOffset + Vector128.Count - offset; + // Store the values into final location + tempLast.StoreUnsafe(ref buf, firstOffset); + tempFirst.StoreUnsafe(ref buf, lastOffset); + } + buf = ref Unsafe.Add(ref buf, numIters * numElements); + length -= numIters * numElements * 2; } - // Store any remaining values one-by-one - if (remainder > 1) - { - ReverseInner(ref Unsafe.Add(ref buf, offset), (nuint)remainder); - } + ReverseInner(ref buf, length); } public static void Reverse(ref long buf, nuint length) { - Debug.Assert(length > 1); - - nint remainder = (nint)length; - nint offset = 0; - - if (Avx2.IsSupported && remainder >= Vector256.Count) + if (Avx2.IsSupported && (nuint)Vector256.Count * 2 <= length) { - nint lastOffset = remainder - Vector256.Count; - do + nuint numElements = (nuint)Vector256.Count; + nuint numIters = (length / numElements) / 2; + for (nuint i = 0; i < numIters; i++) { + nuint firstOffset = i * numElements; + nuint lastOffset = length - ((1 + i) * numElements); // Load the values into vectors - Vector256 tempFirst = Vector256.LoadUnsafe(ref buf, (nuint)offset); - Vector256 tempLast = Vector256.LoadUnsafe(ref buf, (nuint)lastOffset); + Vector256 tempFirst = Vector256.LoadUnsafe(ref buf, firstOffset); + Vector256 tempLast = Vector256.LoadUnsafe(ref buf, lastOffset); // Permute to reverse each vector: // +---------------+ @@ -508,24 +500,24 @@ public static void Reverse(ref long buf, nuint length) tempFirst = Avx2.Permute4x64(tempFirst, 0b00_01_10_11); tempLast = Avx2.Permute4x64(tempLast, 0b00_01_10_11); - // Store the reversed vectors - tempLast.StoreUnsafe(ref buf, (nuint)offset); - tempFirst.StoreUnsafe(ref buf, (nuint)lastOffset); - - offset += Vector256.Count; - lastOffset -= Vector256.Count; - } while (lastOffset >= offset); - - remainder = lastOffset + Vector256.Count - offset; + // Store the values into final location + tempLast.StoreUnsafe(ref buf, firstOffset); + tempFirst.StoreUnsafe(ref buf, lastOffset); + } + buf = ref Unsafe.Add(ref buf, numIters * numElements); + length -= numIters * numElements * 2; } - else if (Vector128.IsHardwareAccelerated && remainder >= Vector128.Count) + else if (Vector128.IsHardwareAccelerated && (nuint)Vector128.Count * 2 <= length) { - nint lastOffset = remainder - Vector128.Count; - do + nuint numElements = (nuint)Vector128.Count; + nuint numIters = (length / numElements) / 2; + for (nuint i = 0; i < numIters; i++) { - // Load in values from beginning and end of the array. - Vector128 tempFirst = Vector128.LoadUnsafe(ref buf, (nuint)offset); - Vector128 tempLast = Vector128.LoadUnsafe(ref buf, (nuint)lastOffset); + nuint firstOffset = i * numElements; + nuint lastOffset = length - ((1 + i) * numElements); + // Load the values into vectors + Vector128 tempFirst = Vector128.LoadUnsafe(ref buf, firstOffset); + Vector128 tempLast = Vector128.LoadUnsafe(ref buf, lastOffset); // Shuffle to reverse each vector: // +-------+ @@ -538,29 +530,22 @@ public static void Reverse(ref long buf, nuint length) tempFirst = Vector128.Shuffle(tempFirst, Vector128.Create(1, 0)); tempLast = Vector128.Shuffle(tempLast, Vector128.Create(1, 0)); - // Store the reversed vectors - tempLast.StoreUnsafe(ref buf, (nuint)offset); - tempFirst.StoreUnsafe(ref buf, (nuint)lastOffset); - - offset += Vector128.Count; - lastOffset -= Vector128.Count; - } while (lastOffset >= offset); - - remainder = lastOffset + Vector128.Count - offset; + // Store the values into final location + tempLast.StoreUnsafe(ref buf, firstOffset); + tempFirst.StoreUnsafe(ref buf, lastOffset); + } + buf = ref Unsafe.Add(ref buf, numIters * numElements); + length -= numIters * (nuint)Vector128.Count * 2; } // Store any remaining values one-by-one - if (remainder > 1) - { - ReverseInner(ref Unsafe.Add(ref buf, offset), (nuint)remainder); - } + ReverseInner(ref buf, length); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Reverse(ref T elements, nuint length) { - Debug.Assert(length > 1); - + Debug.Assert(length > 0); if (!RuntimeHelpers.IsReferenceOrContainsReferences()) { if (Unsafe.SizeOf() == sizeof(byte)) @@ -584,7 +569,6 @@ public static void Reverse(ref T elements, nuint length) return; } } - ReverseInner(ref elements, length); } @@ -592,10 +576,10 @@ public static void Reverse(ref T elements, nuint length) [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void ReverseInner(ref T elements, nuint length) { - Debug.Assert(length > 1); - + if (length <= 1) + return; ref T first = ref elements; - ref T last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, length), 1); + ref T last = ref Unsafe.Subtract(ref Unsafe.Add(ref first, (int)length), 1); do { T temp = first;