Skip to content
This repository has been archived by the owner on Jan 23, 2023. It is now read-only.

Commit

Permalink
Optimize .IndexOfAny
Browse files Browse the repository at this point in the history
  • Loading branch information
@benaadams
benaadams committed Feb 7, 2019
1 parent 991ced6 commit 6ac7daf
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 39 deletions.
47 changes: 29 additions & 18 deletions src/System.Private.CoreLib/shared/System/SpanHelpers.Byte.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -667,11 +667,13 @@ public static unsafe int IndexOfAny(ref byte searchSpace, byte value0, byte valu
do
{
Vector256<byte> search = LoadVector256(ref searchSpace, offset);
// Bitwise Or to combine the matches and MoveMask to convert them to bitflags
int matches = Avx2.MoveMask(
Avx2.Or(
Avx2.CompareEqual(values0, search),
Avx2.CompareEqual(values1, search)));
// Note that MoveMask has converted the equal vector elements into a set of bit flags,
// So the bit position in 'matches' corresponds to the element offset.
int matches = Avx2.MoveMask(Avx2.CompareEqual(values0, search));
// Bitwise Or to combine the flagged matches for the second value to our match flags
matches |= Avx2.MoveMask(Avx2.CompareEqual(values1, search));
if (matches == 0)
{
// Zero flags set so no matches
Expand All @@ -692,8 +694,10 @@ public static unsafe int IndexOfAny(ref byte searchSpace, byte value0, byte valu

Vector128<byte> search = LoadVector128(ref searchSpace, offset);
// Same method as above
int matches = Sse2.MoveMask(Sse2.CompareEqual(values0, search));
matches |= Sse2.MoveMask(Sse2.CompareEqual(values1, search));
int matches = Sse2.MoveMask(
Sse2.Or(
Sse2.CompareEqual(values0, search),
Sse2.CompareEqual(values1, search)));
if (matches == 0)
{
// Zero flags set so no matches
Expand Down Expand Up @@ -726,8 +730,10 @@ public static unsafe int IndexOfAny(ref byte searchSpace, byte value0, byte valu
{
Vector128<byte> search = LoadVector128(ref searchSpace, offset);
// Same method as above
int matches = Sse2.MoveMask(Sse2.CompareEqual(values0, search));
matches |= Sse2.MoveMask(Sse2.CompareEqual(values1, search));
int matches = Sse2.MoveMask(
Sse2.Or(
Sse2.CompareEqual(values0, search),
Sse2.CompareEqual(values1, search)));
if (matches == 0)
{
// Zero flags set so no matches
Expand Down Expand Up @@ -901,13 +907,14 @@ public static unsafe int IndexOfAny(ref byte searchSpace, byte value0, byte valu
do
{
Vector256<byte> search = LoadVector256(ref searchSpace, offset);

Vector256<byte> matches0 = Avx2.CompareEqual(values0, search);
Vector256<byte> matches1 = Avx2.CompareEqual(values1, search);
Vector256<byte> matches2 = Avx2.CompareEqual(values2, search);
// Bitwise Or to combine the matches and MoveMask to convert them to bitflags
int matches = Avx2.MoveMask(Avx2.Or(Avx2.Or(matches0, matches1), matches2));
// Note that MoveMask has converted the equal vector elements into a set of bit flags,
// So the bit position in 'matches' corresponds to the element offset.
int matches = Avx2.MoveMask(Avx2.CompareEqual(values0, search));
// Bitwise Or to combine the flagged matches for the second value to our match flags
matches |= Avx2.MoveMask(Avx2.CompareEqual(values1, search));
// Bitwise Or to combine the flagged matches for the third value to our match flags
matches |= Avx2.MoveMask(Avx2.CompareEqual(values2, search));
if (matches == 0)
{
// Zero flags set so no matches
Expand All @@ -928,10 +935,12 @@ public static unsafe int IndexOfAny(ref byte searchSpace, byte value0, byte valu
Vector128<byte> values2 = Vector128.Create(value2);

Vector128<byte> search = LoadVector128(ref searchSpace, offset);

Vector128<byte> matches0 = Sse2.CompareEqual(values0, search);
Vector128<byte> matches1 = Sse2.CompareEqual(values1, search);
Vector128<byte> matches2 = Sse2.CompareEqual(values2, search);
// Same method as above
int matches = Sse2.MoveMask(Sse2.CompareEqual(values0, search));
matches |= Sse2.MoveMask(Sse2.CompareEqual(values1, search));
matches |= Sse2.MoveMask(Sse2.CompareEqual(values2, search));
int matches = Sse2.MoveMask(Sse2.Or(Sse2.Or(matches0, matches1), matches2));
if (matches == 0)
{
// Zero flags set so no matches
Expand Down Expand Up @@ -964,10 +973,12 @@ public static unsafe int IndexOfAny(ref byte searchSpace, byte value0, byte valu
while ((byte*)lengthToExamine > (byte*)offset)
{
Vector128<byte> search = LoadVector128(ref searchSpace, offset);

Vector128<byte> matches0 = Sse2.CompareEqual(values0, search);
Vector128<byte> matches1 = Sse2.CompareEqual(values1, search);
Vector128<byte> matches2 = Sse2.CompareEqual(values2, search);
// Same method as above
int matches = Sse2.MoveMask(Sse2.CompareEqual(values0, search));
matches |= Sse2.MoveMask(Sse2.CompareEqual(values1, search));
matches |= Sse2.MoveMask(Sse2.CompareEqual(values2, search));
int matches = Sse2.MoveMask(Sse2.Or(Sse2.Or(matches0, matches1), matches2));
if (matches == 0)
{
// Zero flags set so no matches
Expand Down
68 changes: 47 additions & 21 deletions src/System.Private.CoreLib/shared/System/SpanHelpers.Char.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -570,11 +570,14 @@ public static int IndexOfAny(ref char searchSpace, char value0, char value1, int
do
{
Vector256<ushort> search = LoadVector256(ref searchSpace, offset);
// Bitwise Or to combine the flagged matches for the second value to our match flags
int matches = Avx2.MoveMask(
Avx2.Or(
Avx2.CompareEqual(values0, search),
Avx2.CompareEqual(values1, search))
.AsByte());
// Note that MoveMask has converted the equal vector elements into a set of bit flags,
// So the bit position in 'matches' corresponds to the element offset.
int matches = Avx2.MoveMask(Avx2.CompareEqual(values0, search).AsByte());
// Bitwise Or to combine the flagged matches for the second value to our match flags
matches |= Avx2.MoveMask(Avx2.CompareEqual(values1, search).AsByte());
if (matches == 0)
{
// Zero flags set so no matches
Expand All @@ -596,8 +599,11 @@ public static int IndexOfAny(ref char searchSpace, char value0, char value1, int
Vector128<ushort> search = LoadVector128(ref searchSpace, offset);

// Same method as above
int matches = Sse2.MoveMask(Sse2.CompareEqual(values0, search).AsByte());
matches |= Sse2.MoveMask(Sse2.CompareEqual(values1, search).AsByte());
int matches = Sse2.MoveMask(
Sse2.Or(
Sse2.CompareEqual(values0, search),
Sse2.CompareEqual(values1, search))
.AsByte());
if (matches == 0)
{
// Zero flags set so no matches
Expand Down Expand Up @@ -631,8 +637,11 @@ public static int IndexOfAny(ref char searchSpace, char value0, char value1, int
Vector128<ushort> search = LoadVector128(ref searchSpace, offset);

// Same method as above
int matches = Sse2.MoveMask(Sse2.CompareEqual(values0, search).AsByte());
matches |= Sse2.MoveMask(Sse2.CompareEqual(values1, search).AsByte());
int matches = Sse2.MoveMask(
Sse2.Or(
Sse2.CompareEqual(values0, search),
Sse2.CompareEqual(values1, search))
.AsByte());
if (matches == 0)
{
// Zero flags set so no matches
Expand Down Expand Up @@ -767,12 +776,16 @@ public static int IndexOfAny(ref char searchSpace, char value0, char value1, cha
do
{
Vector256<ushort> search = LoadVector256(ref searchSpace, offset);

Vector256<ushort> matches0 = Avx2.CompareEqual(values0, search);
Vector256<ushort> matches1 = Avx2.CompareEqual(values1, search);
Vector256<ushort> matches2 = Avx2.CompareEqual(values2, search);
// Bitwise Or to combine the flagged matches for the second and third values to our match flags
int matches = Avx2.MoveMask(
Avx2.Or(Avx2.Or(matches0, matches1), matches2)
.AsByte());
// Note that MoveMask has converted the equal vector elements into a set of bit flags,
// So the bit position in 'matches' corresponds to the element offset.
int matches = Avx2.MoveMask(Avx2.CompareEqual(values0, search).AsByte());
// Bitwise Or to combine the flagged matches for the second and third values to our match flags
matches |= Avx2.MoveMask(Avx2.CompareEqual(values1, search).AsByte());
matches |= Avx2.MoveMask(Avx2.CompareEqual(values2, search).AsByte());
if (matches == 0)
{
// Zero flags set so no matches
Expand All @@ -792,12 +805,17 @@ public static int IndexOfAny(ref char searchSpace, char value0, char value1, cha
Vector128<ushort> values0 = Vector128.Create(value0);
Vector128<ushort> values1 = Vector128.Create(value1);
Vector128<ushort> values2 = Vector128.Create(value2);

Vector128<ushort> search = LoadVector128(ref searchSpace, offset);

Vector128<ushort> matches0 = Sse2.CompareEqual(values0, search);
Vector128<ushort> matches1 = Sse2.CompareEqual(values1, search);
Vector128<ushort> matches2 = Sse2.CompareEqual(values2, search);

// Same method as above
int matches = Sse2.MoveMask(Sse2.CompareEqual(values0, search).AsByte());
matches |= Sse2.MoveMask(Sse2.CompareEqual(values1, search).AsByte());
matches |= Sse2.MoveMask(Sse2.CompareEqual(values2, search).AsByte());
int matches = Sse2.MoveMask(
Sse2.Or(Sse2.Or(matches0, matches1), matches2)
.AsByte());
if (matches == 0)
{
// Zero flags set so no matches
Expand Down Expand Up @@ -831,10 +849,14 @@ public static int IndexOfAny(ref char searchSpace, char value0, char value1, cha
{
Vector128<ushort> search = LoadVector128(ref searchSpace, offset);

Vector128<ushort> matches0 = Sse2.CompareEqual(values0, search);
Vector128<ushort> matches1 = Sse2.CompareEqual(values1, search);
Vector128<ushort> matches2 = Sse2.CompareEqual(values2, search);

// Same method as above
int matches = Sse2.MoveMask(Sse2.CompareEqual(values0, search).AsByte());
matches |= Sse2.MoveMask(Sse2.CompareEqual(values1, search).AsByte());
matches |= Sse2.MoveMask(Sse2.CompareEqual(values2, search).AsByte());
int matches = Sse2.MoveMask(
Sse2.Or(Sse2.Or(matches0, matches1), matches2)
.AsByte());
if (matches == 0)
{
// Zero flags set so no matches
Expand Down Expand Up @@ -973,13 +995,15 @@ public static int IndexOfAny(ref char searchSpace, char value0, char value1, cha
do
{
Vector256<ushort> search = LoadVector256(ref searchSpace, offset);
// Note that MoveMask has converted the equal vector elements into a set of bit flags,
// So the bit position in 'matches' corresponds to the element offset.
// We preform the Or at non-Vector level as we are using the maximum number of non-preserved registers,
// and more causes them first to be pushed to stack and then popped on exit to preseve their values.
int matches = Avx2.MoveMask(Avx2.CompareEqual(values0, search).AsByte());
// Bitwise Or to combine the flagged matches for the second, third and fourth values to our match flags
matches |= Avx2.MoveMask(Avx2.CompareEqual(values1, search).AsByte());
matches |= Avx2.MoveMask(Avx2.CompareEqual(values2, search).AsByte());
matches |= Avx2.MoveMask(Avx2.CompareEqual(values3, search).AsByte());
// Note that MoveMask has converted the equal vector elements into a set of bit flags,
// So the bit position in 'matches' corresponds to the element offset.
if (matches == 0)
{
// Zero flags set so no matches
Expand Down Expand Up @@ -1186,14 +1210,16 @@ public static int IndexOfAny(ref char searchSpace, char value0, char value1, cha
do
{
Vector256<ushort> search = LoadVector256(ref searchSpace, offset);
// Note that MoveMask has converted the equal vector elements into a set of bit flags,
// So the bit position in 'matches' corresponds to the element offset.
// We preform the Or at non-Vector level as we are using the maximum number of non-preserved registers (+ 1),
// and more causes them first to be pushed to stack and then popped on exit to preseve their values.
int matches = Avx2.MoveMask(Avx2.CompareEqual(values0, search).AsByte());
// Bitwise Or to combine the flagged matches for the second, third, fourth and fifth values to our match flags
matches |= Avx2.MoveMask(Avx2.CompareEqual(values1, search).AsByte());
matches |= Avx2.MoveMask(Avx2.CompareEqual(values2, search).AsByte());
matches |= Avx2.MoveMask(Avx2.CompareEqual(values3, search).AsByte());
matches |= Avx2.MoveMask(Avx2.CompareEqual(values4, search).AsByte());
// Note that MoveMask has converted the equal vector elements into a set of bit flags,
// So the bit position in 'matches' corresponds to the element offset.
if (matches == 0)
{
// Zero flags set so no matches
Expand Down

0 comments on commit 6ac7daf

Please sign in to comment.