Use intrinsics for SequenceCompareTo(byte, ...)

dotnet · Jan 22, 2019 · 342cbe4 · 342cbe4
1 parent aebbd19
commit 342cbe4
Showing 1 changed file with 107 additions and 8 deletions.
diff --git a/src/System.Private.CoreLib/shared/System/SpanHelpers.Byte.cs b/src/System.Private.CoreLib/shared/System/SpanHelpers.Byte.cs
@@ -1384,19 +1384,118 @@ public static unsafe int SequenceCompareTo(ref byte first, int firstLength, ref
             IntPtr i = (IntPtr)0; // Use IntPtr for arithmetic to avoid unnecessary 64->32->64 truncations
             IntPtr n = (IntPtr)(void*)minLength;
 
-            if (Vector.IsHardwareAccelerated && (byte*)n > (byte*)Vector<byte>.Count)
+            if (Avx2.IsSupported)
             {
-                n -= Vector<byte>.Count;
-                while ((byte*)n > (byte*)i)
+                if ((byte*)n >= (byte*)Vector256<byte>.Count)
                 {
-                    if (Unsafe.ReadUnaligned<Vector<byte>>(ref Unsafe.AddByteOffset(ref first, i)) !=
-                        Unsafe.ReadUnaligned<Vector<byte>>(ref Unsafe.AddByteOffset(ref second, i)))
+                    n -= Vector256<byte>.Count;
+                    int matches;
+                    while ((byte*)n > (byte*)i)
                     {
-                        goto NotEqual;
+                        matches = Avx2.MoveMask(Avx2.CompareEqual(
+                            Unsafe.ReadUnaligned<Vector256<byte>>(ref Unsafe.AddByteOffset(ref first, i)), 
+                            Unsafe.ReadUnaligned<Vector256<byte>>(ref Unsafe.AddByteOffset(ref second, i))));
+                        if (matches == -1)
+                        {
+                            // All matched
+                            i += Vector256<byte>.Count;
+                            continue;
+                        }
+
+                        goto Difference;
                     }
-                    i += Vector<byte>.Count;
+                    // Move to Vector length from end for final compare
+                    i = n;
+                    matches = Avx2.MoveMask(Avx2.CompareEqual(
+                            Unsafe.ReadUnaligned<Vector256<byte>>(ref Unsafe.AddByteOffset(ref first, i)),
+                            Unsafe.ReadUnaligned<Vector256<byte>>(ref Unsafe.AddByteOffset(ref second, i))));
+                    if (matches == -1)
+                    {
+                        // All matched
+                        goto Equal;
+                    }
+                Difference:
+                    // Invert matches to find differences
+                    int differences = ~matches;
+                    if (Bmi1.IsSupported)
+                    {
+                        i = (IntPtr)((int)(byte*)i) + (int)Bmi1.TrailingZeroCount((uint)differences);
+                    }
+                    else
+                    {
+                        i = (IntPtr)(int)(byte*)i + TrailingZeroCountFallback(differences);
+                    }
+
+                    int result = Unsafe.AddByteOffset(ref first, i).CompareTo(Unsafe.AddByteOffset(ref second, i));
+                    Debug.Assert(result != 0);
+
+                    return result;
+                }
+            }
+            else if (Sse2.IsSupported)
+            {
+                if ((byte*)n >= (byte*)Vector128<byte>.Count)
+                {
+                    n -= Vector128<byte>.Count;
+                    int matches;
+                    while ((byte*)n > (byte*)i)
+                    {
+                        matches = Sse2.MoveMask(Sse2.CompareEqual(
+                            Unsafe.ReadUnaligned<Vector128<byte>>(ref Unsafe.AddByteOffset(ref first, i)),
+                            Unsafe.ReadUnaligned<Vector128<byte>>(ref Unsafe.AddByteOffset(ref second, i))));
+                        if (matches == 0xFFFF)
+                        {
+                            // All matched
+                            i += Vector128<byte>.Count;
+                            continue;
+                        }
+
+                        goto Difference;
+                    }
+                    // Move to Vector length from end for final compare
+                    i = n;
+                    matches = Sse2.MoveMask(Sse2.CompareEqual(
+                            Unsafe.ReadUnaligned<Vector128<byte>>(ref Unsafe.AddByteOffset(ref first, i)),
+                            Unsafe.ReadUnaligned<Vector128<byte>>(ref Unsafe.AddByteOffset(ref second, i))));
+                    if (matches == 0xFFFF)
+                    {
+                        // All matched
+                        goto Equal;
+                    }
+                Difference:
+                    // Invert matches to find differences
+                    int differences = ~matches;
+                    if (Bmi1.IsSupported)
+                    {
+                        i = (IntPtr)((int)(byte*)i) + (int)Bmi1.TrailingZeroCount((uint)differences);
+                    }
+                    else
+                    {
+                        i = (IntPtr)(int)(byte*)i + TrailingZeroCountFallback(differences);
+                    }
+
+                    int result = Unsafe.AddByteOffset(ref first, i).CompareTo(Unsafe.AddByteOffset(ref second, i));
+                    Debug.Assert(result != 0);
+
+                    return result;
+                }
+            }
+            else if (Vector.IsHardwareAccelerated)
+            {
+                if ((byte*) n > (byte*)Vector<byte>.Count)
+                {
+                    n -= Vector<byte>.Count;
+                    while ((byte*)n > (byte*)i)
+                    {
+                        if (Unsafe.ReadUnaligned<Vector<byte>>(ref Unsafe.AddByteOffset(ref first, i)) !=
+                            Unsafe.ReadUnaligned<Vector<byte>>(ref Unsafe.AddByteOffset(ref second, i)))
+                        {
+                            goto NotEqual;
+                        }
+                        i += Vector<byte>.Count;
+                    }
+                    goto NotEqual;
                 }
-                goto NotEqual;
             }
 
             if ((byte*)n > (byte*)sizeof(UIntPtr))