Vectorize ProbabilisticMap.IndexOfAny (#80963)

* Vectorize ProbabilisticMap.IndexOfAny on AVX2 * Use ResetLowestSetBit from BitOperations * Speed up Avx2 and add Vector128 support * Add Vector{128/256}.LoadUnsafe(ref char) and Vector128.ShuffleUnsafe * Use Vector128.ShuffleUnsafe in more places * PR feedback * Replace another ShiftRightLogical with '>>>' * Add WASM path to Vector128.ShuffleUnsafe * PR feedback
dotnet · Mar 10, 2023 · 7a0b0e1 · 7a0b0e1
1 parent 2c4059e
commit 7a0b0e1
Show file tree

Hide file tree

Showing 9 changed files with 313 additions and 54 deletions.
diff --git a/src/libraries/Common/src/System/HexConverter.cs b/src/libraries/Common/src/System/HexConverter.cs
@@ -99,13 +99,8 @@ internal static (Vector128<byte>, Vector128<byte>) AsciiToHexVector128(Vector128
             Vector128<byte> lowNibbles = Vector128.UnpackLow(shiftedSrc, src);
             Vector128<byte> highNibbles = Vector128.UnpackHigh(shiftedSrc, src);
 
-            return (ShuffleUnsafe(hexMap, lowNibbles & Vector128.Create((byte)0xF)),
-                ShuffleUnsafe(hexMap, highNibbles & Vector128.Create((byte)0xF)));
-
-            // TODO: remove once https://github.com/dotnet/runtime/pull/80963 is merged
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            static Vector128<byte> ShuffleUnsafe(Vector128<byte> value, Vector128<byte> mask)
-                => Ssse3.IsSupported ? Ssse3.Shuffle(value, mask) : AdvSimd.Arm64.VectorTableLookup(value, mask);
+            return (Vector128.ShuffleUnsafe(hexMap, lowNibbles & Vector128.Create((byte)0xF)),
+                Vector128.ShuffleUnsafe(hexMap, highNibbles & Vector128.Create((byte)0xF)));
         }
 
         private static void EncodeToUtf16_Vector128(ReadOnlySpan<byte> bytes, Span<char> chars, Casing casing)

diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Decoder.cs b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Decoder.cs
@@ -477,20 +477,17 @@ private static unsafe void Avx2Decode(ref byte* srcBytes, ref byte* destBytes, b
             destBytes = dest;
         }
 
-        // This can be replaced once https://github.com/dotnet/runtime/issues/63331 is implemented.
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static Vector128<byte> SimdShuffle(Vector128<byte> left, Vector128<byte> right, Vector128<byte> mask8F)
         {
             Debug.Assert((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported) && BitConverter.IsLittleEndian);
 
-            if (Ssse3.IsSupported)
+            if (AdvSimd.Arm64.IsSupported)
             {
-                return Ssse3.Shuffle(left, right);
-            }
-            else
-            {
-                return AdvSimd.Arm64.VectorTableLookup(left, Vector128.BitwiseAnd(right, mask8F));
+                right &= mask8F;
             }
+
+            return Vector128.ShuffleUnsafe(left, right);
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]

diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs
@@ -88,8 +88,8 @@ private static bool EqualsIgnoreCase_Vector128(ref char charA, ref char charB, i
             Vector128<ushort> vec2;
             do
             {
-                vec1 = Vector128.LoadUnsafe(ref Unsafe.As<char, ushort>(ref charA), i);
-                vec2 = Vector128.LoadUnsafe(ref Unsafe.As<char, ushort>(ref charB), i);
+                vec1 = Vector128.LoadUnsafe(ref charA, i);
+                vec2 = Vector128.LoadUnsafe(ref charB, i);
 
                 if (!Utf16Utility.AllCharsInVector128AreAscii(vec1 | vec2))
                 {

diff --git a/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyAsciiSearcher.cs b/src/libraries/System.Private.CoreLib/src/System/IndexOfAnyValues/IndexOfAnyAsciiSearcher.cs
@@ -860,10 +860,10 @@ private static Vector128<byte> IndexOfAnyLookupCore(Vector128<byte> source, Vect
 
             // The bitmapLookup represents a 8x16 table of bits, indicating whether a character is present in the needle.
             // Lookup the rows via the lower nibble and the column via the higher nibble.
-            Vector128<byte> bitMask = Shuffle(bitmapLookup, lowNibbles);
+            Vector128<byte> bitMask = Vector128.ShuffleUnsafe(bitmapLookup, lowNibbles);
 
             // For values above 127, the high nibble will be above 7. We construct the positions vector for the shuffle such that those values map to 0.
-            Vector128<byte> bitPositions = Shuffle(Vector128.Create(0x8040201008040201, 0).AsByte(), highNibbles);
+            Vector128<byte> bitPositions = Vector128.ShuffleUnsafe(Vector128.Create(0x8040201008040201, 0).AsByte(), highNibbles);
 
             Vector128<byte> result = bitMask & bitPositions;
             return result;
@@ -909,10 +909,10 @@ private static Vector128<byte> IndexOfAnyLookup<TNegator>(Vector128<byte> source
             Vector128<byte> lowNibbles = source & Vector128.Create((byte)0xF);
             Vector128<byte> highNibbles = Vector128.ShiftRightLogical(source.AsInt32(), 4).AsByte() & Vector128.Create((byte)0xF);
 
-            Vector128<byte> row0 = Shuffle(bitmapLookup0, lowNibbles);
-            Vector128<byte> row1 = Shuffle(bitmapLookup1, lowNibbles);
+            Vector128<byte> row0 = Vector128.ShuffleUnsafe(bitmapLookup0, lowNibbles);
+            Vector128<byte> row1 = Vector128.ShuffleUnsafe(bitmapLookup1, lowNibbles);
 
-            Vector128<byte> bitmask = Shuffle(Vector128.Create(0x8040201008040201).AsByte(), highNibbles);
+            Vector128<byte> bitmask = Vector128.ShuffleUnsafe(Vector128.Create(0x8040201008040201).AsByte(), highNibbles);
 
             Vector128<byte> mask = Vector128.GreaterThan(highNibbles.AsSByte(), Vector128.Create((sbyte)0x7)).AsByte();
             Vector128<byte> bitsets = Vector128.ConditionalSelect(mask, row1, row0);
@@ -944,16 +944,6 @@ private static Vector256<byte> IndexOfAnyLookup<TNegator>(Vector256<byte> source
             return TNegator.NegateIfNeeded(result);
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static Vector128<byte> Shuffle(Vector128<byte> vector, Vector128<byte> indices)
-        {
-            // We're not using Vector128.Shuffle as the caller already accounts for and relies on differences in behavior between platforms.
-            return
-                Ssse3.IsSupported ? Ssse3.Shuffle(vector, indices) :
-                AdvSimd.Arm64.IsSupported ? AdvSimd.Arm64.VectorTableLookup(vector, indices) :
-                PackedSimd.Swizzle(vector, indices);
-        }
-
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static unsafe int ComputeFirstIndex<T, TNegator>(ref T searchSpace, ref T current, Vector128<byte> result)
             where TNegator : struct, INegator