Skip to content

Commit

Permalink
[Arm64] Implement Vector64/128.CreateScalar() using AdvSimd.Insert (#…
Browse files Browse the repository at this point in the history 
…35300)
  • Loading branch information
@echesakov
echesakov authored Apr 30, 2020
1 parent 2ae694d commit 670bf21
Show file tree
Hide file tree
Showing 2 changed files with 143 additions and 21 deletions.
51 changes: 51 additions & 0 deletions src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector128.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Diagnostics;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.Intrinsics.Arm;
using System.Runtime.Intrinsics.X86;
using Internal.Runtime.CompilerServices;

Expand Down Expand Up @@ -1369,6 +1370,11 @@ public static unsafe Vector128<ulong> Create(Vector64<ulong> lower, Vector64<ulo
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static unsafe Vector128<byte> CreateScalar(byte value)
{
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector128<byte>.Zero, 0, value);
}

if (Sse2.IsSupported)
{
// ConvertScalarToVector128 only deals with 32/64-bit inputs and we need to ensure all upper-bits are zeroed, so we call
Expand All @@ -1392,6 +1398,11 @@ static Vector128<byte> SoftwareFallback(byte value)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static unsafe Vector128<double> CreateScalar(double value)
{
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector128<double>.Zero, 0, value);
}

if (Sse2.IsSupported)
{
return Sse2.MoveScalar(Vector128<double>.Zero, CreateScalarUnsafe(value));
Expand All @@ -1413,6 +1424,11 @@ static Vector128<double> SoftwareFallback(double value)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static unsafe Vector128<short> CreateScalar(short value)
{
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector128<short>.Zero, 0, value);
}

if (Sse2.IsSupported)
{
// ConvertScalarToVector128 only deals with 32/64-bit inputs and we need to ensure all upper-bits are zeroed, so we cast
Expand All @@ -1436,6 +1452,11 @@ static Vector128<short> SoftwareFallback(short value)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static unsafe Vector128<int> CreateScalar(int value)
{
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector128<int>.Zero, 0, value);
}

if (Sse2.IsSupported)
{
return Sse2.ConvertScalarToVector128Int32(value);
Expand All @@ -1456,6 +1477,11 @@ static Vector128<int> SoftwareFallback(int value)
/// <returns>A new <see cref="Vector128{Int64}" /> instance with the first element initialized to <paramref name="value" /> and the remaining elements initialized to zero.</returns>
public static unsafe Vector128<long> CreateScalar(long value)
{
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector128<long>.Zero, 0, value);
}

if (Sse2.X64.IsSupported)
{
return Sse2.X64.ConvertScalarToVector128Int64(value);
Expand All @@ -1478,6 +1504,11 @@ static Vector128<long> SoftwareFallback(long value)
[CLSCompliant(false)]
public static unsafe Vector128<sbyte> CreateScalar(sbyte value)
{
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector128<sbyte>.Zero, 0, value);
}

if (Sse2.IsSupported)
{
// ConvertScalarToVector128 only deals with 32/64-bit inputs and we need to ensure all upper-bits are zeroed, so we cast
Expand All @@ -1501,6 +1532,11 @@ static Vector128<sbyte> SoftwareFallback(sbyte value)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static unsafe Vector128<float> CreateScalar(float value)
{
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector128<float>.Zero, 0, value);
}

if (Sse.IsSupported)
{
return Sse.MoveScalar(Vector128<float>.Zero, CreateScalarUnsafe(value));
Expand All @@ -1523,6 +1559,11 @@ static Vector128<float> SoftwareFallback(float value)
[CLSCompliant(false)]
public static unsafe Vector128<ushort> CreateScalar(ushort value)
{
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector128<ushort>.Zero, 0, value);
}

if (Sse2.IsSupported)
{
// ConvertScalarToVector128 only deals with 32/64-bit inputs and we need to ensure all upper-bits are zeroed, so we call
Expand All @@ -1547,6 +1588,11 @@ static Vector128<ushort> SoftwareFallback(ushort value)
[CLSCompliant(false)]
public static unsafe Vector128<uint> CreateScalar(uint value)
{
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector128<uint>.Zero, 0, value);
}

if (Sse2.IsSupported)
{
return Sse2.ConvertScalarToVector128UInt32(value);
Expand All @@ -1569,6 +1615,11 @@ static Vector128<uint> SoftwareFallback(uint value)
[CLSCompliant(false)]
public static unsafe Vector128<ulong> CreateScalar(ulong value)
{
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector128<ulong>.Zero, 0, value);
}

if (Sse2.X64.IsSupported)
{
return Sse2.X64.ConvertScalarToVector128UInt64(value);
Expand Down
113 changes: 92 additions & 21 deletions src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector64.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

using System.Runtime.CompilerServices;
using Internal.Runtime.CompilerServices;
using System.Runtime.Intrinsics.Arm;

namespace System.Runtime.Intrinsics
{
Expand Down Expand Up @@ -451,29 +452,59 @@ public static unsafe Vector64<uint> Create(uint e0, uint e1)
/// <returns>A new <see cref="Vector64{Byte}" /> instance with the first element initialized to <paramref name="value"/> and the remaining elements initialized to zero.</returns>
public static unsafe Vector64<byte> CreateScalar(byte value)
{
var result = Vector64<byte>.Zero;
Unsafe.WriteUnaligned(ref Unsafe.As<Vector64<byte>, byte>(ref result), value);
return result;
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector64<byte>.Zero, 0, value);
}

return SoftwareFallback(value);

static Vector64<byte> SoftwareFallback(byte value)
{
var result = Vector64<byte>.Zero;
Unsafe.WriteUnaligned(ref Unsafe.As<Vector64<byte>, byte>(ref result), value);
return result;
}
}

/// <summary>Creates a new <see cref="Vector64{Int16}" /> instance with the first element initialized to the specified value and the remaining elements initialized to zero.</summary>
/// <param name="value">The value that element 0 will be initialized to.</param>
/// <returns>A new <see cref="Vector64{Int16}" /> instance with the first element initialized to <paramref name="value"/> and the remaining elements initialized to zero.</returns>
public static unsafe Vector64<short> CreateScalar(short value)
{
var result = Vector64<short>.Zero;
Unsafe.WriteUnaligned(ref Unsafe.As<Vector64<short>, byte>(ref result), value);
return result;
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector64<short>.Zero, 0, value);
}

return SoftwareFallback(value);

static Vector64<short> SoftwareFallback(short value)
{
var result = Vector64<short>.Zero;
Unsafe.WriteUnaligned(ref Unsafe.As<Vector64<short>, byte>(ref result), value);
return result;
}
}

/// <summary>Creates a new <see cref="Vector64{Int32}" /> instance with the first element initialized to the specified value and the remaining elements initialized to zero.</summary>
/// <param name="value">The value that element 0 will be initialized to.</param>
/// <returns>A new <see cref="Vector64{Int32}" /> instance with the first element initialized to <paramref name="value"/> and the remaining elements initialized to zero.</returns>
public static unsafe Vector64<int> CreateScalar(int value)
{
var result = Vector64<int>.Zero;
Unsafe.WriteUnaligned(ref Unsafe.As<Vector64<int>, byte>(ref result), value);
return result;
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector64<int>.Zero, 0, value);
}

return SoftwareFallback(value);

static Vector64<int> SoftwareFallback(int value)
{
var result = Vector64<int>.Zero;
Unsafe.WriteUnaligned(ref Unsafe.As<Vector64<int>, byte>(ref result), value);
return result;
}
}

/// <summary>Creates a new <see cref="Vector64{SByte}" /> instance with the first element initialized to the specified value and the remaining elements initialized to zero.</summary>
Expand All @@ -482,19 +513,39 @@ public static unsafe Vector64<int> CreateScalar(int value)
[CLSCompliant(false)]
public static unsafe Vector64<sbyte> CreateScalar(sbyte value)
{
var result = Vector64<sbyte>.Zero;
Unsafe.WriteUnaligned(ref Unsafe.As<Vector64<sbyte>, byte>(ref result), value);
return result;
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector64<sbyte>.Zero, 0, value);
}

return SoftwareFallback(value);

static Vector64<sbyte> SoftwareFallback(sbyte value)
{
var result = Vector64<sbyte>.Zero;
Unsafe.WriteUnaligned(ref Unsafe.As<Vector64<sbyte>, byte>(ref result), value);
return result;
}
}

/// <summary>Creates a new <see cref="Vector64{Single}" /> instance with the first element initialized to the specified value and the remaining elements initialized to zero.</summary>
/// <param name="value">The value that element 0 will be initialized to.</param>
/// <returns>A new <see cref="Vector64{Single}" /> instance with the first element initialized to <paramref name="value"/> and the remaining elements initialized to zero.</returns>
public static unsafe Vector64<float> CreateScalar(float value)
{
var result = Vector64<float>.Zero;
Unsafe.WriteUnaligned(ref Unsafe.As<Vector64<float>, byte>(ref result), value);
return result;
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector64<float>.Zero, 0, value);
}

return SoftwareFallback(value);

static Vector64<float> SoftwareFallback(float value)
{
var result = Vector64<float>.Zero;
Unsafe.WriteUnaligned(ref Unsafe.As<Vector64<float>, byte>(ref result), value);
return result;
}
}

/// <summary>Creates a new <see cref="Vector64{UInt16}" /> instance with the first element initialized to the specified value and the remaining elements initialized to zero.</summary>
Expand All @@ -503,9 +554,19 @@ public static unsafe Vector64<float> CreateScalar(float value)
[CLSCompliant(false)]
public static unsafe Vector64<ushort> CreateScalar(ushort value)
{
var result = Vector64<ushort>.Zero;
Unsafe.WriteUnaligned(ref Unsafe.As<Vector64<ushort>, byte>(ref result), value);
return result;
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector64<ushort>.Zero, 0, value);
}

return SoftwareFallback(value);

static Vector64<ushort> SoftwareFallback(ushort value)
{
var result = Vector64<ushort>.Zero;
Unsafe.WriteUnaligned(ref Unsafe.As<Vector64<ushort>, byte>(ref result), value);
return result;
}
}

/// <summary>Creates a new <see cref="Vector64{UInt32}" /> instance with the first element initialized to the specified value and the remaining elements initialized to zero.</summary>
Expand All @@ -514,9 +575,19 @@ public static unsafe Vector64<ushort> CreateScalar(ushort value)
[CLSCompliant(false)]
public static unsafe Vector64<uint> CreateScalar(uint value)
{
var result = Vector64<uint>.Zero;
Unsafe.WriteUnaligned(ref Unsafe.As<Vector64<uint>, byte>(ref result), value);
return result;
if (AdvSimd.IsSupported)
{
return AdvSimd.Insert(Vector64<uint>.Zero, 0, value);
}

return SoftwareFallback(value);

static Vector64<uint> SoftwareFallback(uint value)
{
var result = Vector64<uint>.Zero;
Unsafe.WriteUnaligned(ref Unsafe.As<Vector64<uint>, byte>(ref result), value);
return result;
}
}

/// <summary>Creates a new <see cref="Vector64{Byte}" /> instance with the first element initialized to the specified value and the remaining elements left uninitialized.</summary>
Expand Down

0 comments on commit 670bf21

Please sign in to comment.