From 3a294edd07d2840567138a87abc45e55f19b460b Mon Sep 17 00:00:00 2001 From: Will Smith Date: Fri, 28 Jun 2024 21:17:33 -0700 Subject: [PATCH] JIT: Added SVE APIs `CreateMaskForFirstActiveElement` and `CreateMaskForNextActiveElement` (#104002) * Initial work * Added tests. Fixed parameter names. * Use delay free for op1 if the target preference is op2. Use sve_mov instead of mov. * Feedback * Feedback * Update Helpers.cs * Handle RMW for non-explicit masked operation * Remove handling as its already handled it looks like * Feedback * Feedback * Feedback --- src/coreclr/jit/hwintrinsic.cpp | 2 + src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 33 ++- src/coreclr/jit/hwintrinsiclistarm64sve.h | 2 + src/coreclr/jit/instr.cpp | 2 - src/coreclr/jit/lsraarm64.cpp | 9 +- .../Arm/Sve.PlatformNotSupported.cs | 73 ++++++ .../src/System/Runtime/Intrinsics/Arm/Sve.cs | 73 ++++++ .../ref/System.Runtime.Intrinsics.cs | 14 ++ .../GenerateHWIntrinsicTests_Arm.cs | 14 ++ .../HardwareIntrinsics/Arm/Shared/Helpers.cs | 236 ++++++++++++++++++ 10 files changed, 454 insertions(+), 4 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 230cd7ff147dc..c78f712433cb0 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1922,6 +1922,8 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, break; } + case NI_Sve_CreateMaskForFirstActiveElement: + case NI_Sve_CreateMaskForNextActiveElement: case NI_Sve_GetActiveElementCount: case NI_Sve_TestAnyTrue: case NI_Sve_TestFirstTrue: diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 549e6f2f3ea46..9a2e8e810f5d1 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -867,7 +867,23 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) assert(!node->IsEmbMaskOp()); if (HWIntrinsicInfo::IsExplicitMaskedOperation(intrin.id)) { - GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt); + if (isRMW) + { + if (targetReg != op2Reg) + { + assert(targetReg != op1Reg); + + GetEmitter()->emitIns_Mov(ins_Move_Extend(intrin.op2->TypeGet(), false), + emitTypeSize(node), targetReg, op2Reg, + /* canSkip */ true); + } + + GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op1Reg, opt); + } + else + { + GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt); + } } else { @@ -2211,6 +2227,21 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Sve_CreateMaskForFirstActiveElement: + { + assert(isRMW); + assert(HWIntrinsicInfo::IsExplicitMaskedOperation(intrin.id)); + + if (targetReg != op2Reg) + { + assert(targetReg != op1Reg); + GetEmitter()->emitIns_Mov(INS_sve_mov, emitTypeSize(node), targetReg, op2Reg, /* canSkip */ true); + } + + GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op1Reg, INS_OPTS_SCALABLE_B); + break; + } + default: unreached(); } diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 5c734c55ddeaa..e9b705ea20f47 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -47,6 +47,8 @@ HARDWARE_INTRINSIC(Sve, CreateFalseMaskSingle, HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt16, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt32, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateFalseMaskUInt64, -1, 0, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_pfalse, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) +HARDWARE_INTRINSIC(Sve, CreateMaskForFirstActiveElement, -1, 2, true, {INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_sve_pfirst, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) +HARDWARE_INTRINSIC(Sve, CreateMaskForNextActiveElement, -1, 2, true, {INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_sve_pnext, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_ReturnsPerElementMask|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve, CreateTrueMaskByte, -1, 1, false, {INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateTrueMaskDouble, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_ReturnsPerElementMask) HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt16, -1, 1, false, {INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_ReturnsPerElementMask) diff --git a/src/coreclr/jit/instr.cpp b/src/coreclr/jit/instr.cpp index f8c05010f22c2..382350ebfc5fc 100644 --- a/src/coreclr/jit/instr.cpp +++ b/src/coreclr/jit/instr.cpp @@ -1721,7 +1721,6 @@ instruction CodeGen::ins_Move_Extend(var_types srcType, bool srcInReg) #if defined(TARGET_XARCH) return INS_kmovq_msk; #elif defined(TARGET_ARM64) - unreached(); // TODO-SVE: This needs testing return INS_sve_mov; #endif } @@ -2085,7 +2084,6 @@ instruction CodeGen::ins_Copy(regNumber srcReg, var_types dstType) #if defined(TARGET_XARCH) return INS_kmovq_gpr; #elif defined(TARGET_ARM64) - unreached(); // TODO-SVE: This needs testing return INS_sve_mov; #endif } diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 4c4c58fd00bf9..c344596f632e5 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1627,7 +1627,14 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou predMask = RBM_LOWMASK.GetPredicateRegSet(); } - srcCount += BuildOperandUses(intrin.op1, predMask); + if (tgtPrefOp2) + { + srcCount += BuildDelayFreeUses(intrin.op1, intrin.op2, predMask); + } + else + { + srcCount += BuildOperandUses(intrin.op1, predMask); + } } } else if (intrinsicTree->OperIsMemoryLoadOrStore()) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index 6495d82f46b75..71734f02404f1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -1016,6 +1016,79 @@ internal Arm64() { } public static unsafe Vector CreateFalseMaskUInt64() { throw new PlatformNotSupportedException(); } + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpnext_b8(svbool_t pg, svbool_t op) + /// PNEXT Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpnext_b16(svbool_t pg, svbool_t op) + /// PNEXT Ptied.H, Pg, Ptied.H + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpnext_b32(svbool_t pg, svbool_t op) + /// PNEXT Ptied.S, Pg, Ptied.S + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } + + /// + /// svbool_t svpnext_b64(svbool_t pg, svbool_t op) + /// PNEXT Ptied.D, Pg, Ptied.D + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) { throw new PlatformNotSupportedException(); } + + /// CreateTrueMaskByte : Set predicate elements to true /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index 032e6dd2290fd..83808ac6374d5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -1073,6 +1073,79 @@ internal Arm64() { } public static unsafe Vector CreateFalseMaskUInt64() => CreateFalseMaskUInt64(); + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); + + /// + /// svbool_t svpfirst[_b](svbool_t pg, svbool_t op) + /// PFIRST Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForFirstActiveElement(Vector mask, Vector srcMask) => CreateMaskForFirstActiveElement(mask, srcMask); + + /// + /// svbool_t svpnext_b8(svbool_t pg, svbool_t op) + /// PNEXT Ptied.B, Pg, Ptied.B + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) => CreateMaskForNextActiveElement(mask, srcMask); + + /// + /// svbool_t svpnext_b16(svbool_t pg, svbool_t op) + /// PNEXT Ptied.H, Pg, Ptied.H + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) => CreateMaskForNextActiveElement(mask, srcMask); + + /// + /// svbool_t svpnext_b32(svbool_t pg, svbool_t op) + /// PNEXT Ptied.S, Pg, Ptied.S + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) => CreateMaskForNextActiveElement(mask, srcMask); + + /// + /// svbool_t svpnext_b64(svbool_t pg, svbool_t op) + /// PNEXT Ptied.D, Pg, Ptied.D + /// + public static unsafe Vector CreateMaskForNextActiveElement(Vector mask, Vector srcMask) => CreateMaskForNextActiveElement(mask, srcMask); + + /// CreateTrueMaskByte : Set predicate elements to true /// diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 188be8b482fb6..b18efc40efe73 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -4335,6 +4335,20 @@ internal Arm64() { } public static System.Numerics.Vector CreateFalseMaskUInt16() { throw null; } public static System.Numerics.Vector CreateFalseMaskUInt32() { throw null; } public static System.Numerics.Vector CreateFalseMaskUInt64() { throw null; } + + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForFirstActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static unsafe System.Numerics.Vector CreateMaskForNextActiveElement(System.Numerics.Vector mask, System.Numerics.Vector srcMask) { throw null; } + public static System.Numerics.Vector CreateTrueMaskByte([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } public static System.Numerics.Vector CreateTrueMaskDouble([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } public static System.Numerics.Vector CreateTrueMaskInt16([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index 57668d99c0212..5cfe4287ee8d0 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -3192,6 +3192,20 @@ ("SveVecImmUnOpTest.template", new Dictionary { ["TestName"] = "Sve_DuplicateSelectedScalarToVector_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "DuplicateSelectedScalarToVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["InvalidImm"] = "16", ["Imm"] = "TestLibrary.Generator.GetByte() % 16", ["ValidateIterResult"] = "result[i] != (imm < Op1ElementCount ? firstOp[imm] : 0)", ["GetIterResult"] = "(UInt32)(imm < Op1ElementCount ? firstOp[imm] : 0)"}), ("SveVecImmUnOpTest.template", new Dictionary { ["TestName"] = "Sve_DuplicateSelectedScalarToVector_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "DuplicateSelectedScalarToVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["InvalidImm"] = "8", ["Imm"] = "TestLibrary.Generator.GetByte() % 8", ["ValidateIterResult"] = "result[i] != (imm < Op1ElementCount ? firstOp[imm] : 0)", ["GetIterResult"] = "(UInt64)(imm < Op1ElementCount ? firstOp[imm] : 0)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskSByte()", ["NextValueOp2"] = "Helpers.getMaskSByte()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt16()", ["NextValueOp2"] = "Helpers.getMaskInt16()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt32()", ["NextValueOp2"] = "Helpers.getMaskInt32()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskInt64()", ["NextValueOp2"] = "Helpers.getMaskInt64()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskByte()", ["NextValueOp2"] = "Helpers.getMaskByte()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt16()", ["NextValueOp2"] = "Helpers.getMaskUInt16()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "Helpers.getMaskUInt32()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForFirstActiveElement_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForFirstActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "Helpers.getMaskUInt64()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForFirstActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForFirstActiveElement(left, right)"}), + + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForNextActiveElement_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForNextActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskByte()", ["NextValueOp2"] = "Helpers.getMaskByte()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForNextActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForNextActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForNextActiveElement_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForNextActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt16()", ["NextValueOp2"] = "Helpers.getMaskUInt16()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForNextActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForNextActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForNextActiveElement_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForNextActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt32()", ["NextValueOp2"] = "Helpers.getMaskUInt32()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForNextActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForNextActiveElement(left, right)"}), + ("SveVecBinOpVecTest.template", new Dictionary { ["TestName"] = "Sve_CreateMaskForNextActiveElement_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "CreateMaskForNextActiveElement", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "Helpers.getMaskUInt64()", ["NextValueOp2"] = "Helpers.getMaskUInt64()", ["ValidateVectorResult"] = "!result.SequenceEqual(Helpers.CreateMaskForNextActiveElement(left, right))", ["GetVectorResult"] = "Helpers.CreateMaskForNextActiveElement(left, right)"}), + ("SveExtractVectorTest.template", new Dictionary { ["TestName"] = "SveExtractVector_Byte_1", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ExtractVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ElementIndex"] = "1", ["ValidateIterResult"] = "Helpers.ExtractVector(firstOp, secondOp, ElementIndex, i) != result[i]"}), ("SveExtractVectorTest.template", new Dictionary { ["TestName"] = "SveExtractVector_Double_1", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ExtractVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ElementIndex"] = "1", ["ValidateIterResult"] = "BitConverter.DoubleToInt64Bits(Helpers.ExtractVector(firstOp, secondOp, ElementIndex, i)) != BitConverter.DoubleToInt64Bits(result[i])"}), ("SveExtractVectorTest.template", new Dictionary { ["TestName"] = "SveExtractVector_Int16_1", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ExtractVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ElementIndex"] = "1", ["ValidateIterResult"] = "Helpers.ExtractVector(firstOp, secondOp, ElementIndex, i) != result[i]"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index 43f149787e0dc..4c9f4a3fbed79 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -28,6 +28,242 @@ public static Vector InitVector(Func f) return new Vector(arr); } + public static byte[] CreateMaskForFirstActiveElement(byte[] mask, byte[] srcMask) + { + var count = srcMask.Length; + var result = new byte[count]; + Array.Copy(srcMask, 0, result, 0, count); + for (var i = 0; i < count; i++) + { + if (mask[i] != 0) + { + result[i] = 1; + return result; + } + } + return result; + } + + public static short[] CreateMaskForFirstActiveElement(short[] mask, short[] srcMask) + { + var count = srcMask.Length; + var result = new short[count]; + Array.Copy(srcMask, 0, result, 0, count); + for (var i = 0; i < count; i++) + { + if (mask[i] != 0) + { + result[i] = 1; + return result; + } + } + return result; + } + + public static int[] CreateMaskForFirstActiveElement(int[] mask, int[] srcMask) + { + var count = srcMask.Length; + var result = new int[count]; + Array.Copy(srcMask, 0, result, 0, count); + for (var i = 0; i < count; i++) + { + if (mask[i] != 0) + { + result[i] = 1; + return result; + } + } + return result; + } + + public static long[] CreateMaskForFirstActiveElement(long[] mask, long[] srcMask) + { + var count = srcMask.Length; + var result = new long[count]; + Array.Copy(srcMask, 0, result, 0, count); + for (var i = 0; i < count; i++) + { + if (mask[i] != 0) + { + result[i] = 1; + return result; + } + } + return result; + } + + public static sbyte[] CreateMaskForFirstActiveElement(sbyte[] mask, sbyte[] srcMask) + { + var count = srcMask.Length; + var result = new sbyte[count]; + Array.Copy(srcMask, 0, result, 0, count); + for (var i = 0; i < count; i++) + { + if (mask[i] != 0) + { + result[i] = 1; + return result; + } + } + return result; + } + + public static ushort[] CreateMaskForFirstActiveElement(ushort[] mask, ushort[] srcMask) + { + var count = srcMask.Length; + var result = new ushort[count]; + Array.Copy(srcMask, 0, result, 0, count); + for (var i = 0; i < count; i++) + { + if (mask[i] != 0) + { + result[i] = 1; + return result; + } + } + return result; + } + + public static uint[] CreateMaskForFirstActiveElement(uint[] mask, uint[] srcMask) + { + var count = srcMask.Length; + var result = new uint[count]; + Array.Copy(srcMask, 0, result, 0, count); + for (var i = 0; i < count; i++) + { + if (mask[i] != 0) + { + result[i] = 1; + return result; + } + } + return result; + } + + public static ulong[] CreateMaskForFirstActiveElement(ulong[] mask, ulong[] srcMask) + { + var count = srcMask.Length; + var result = new ulong[count]; + Array.Copy(srcMask, 0, result, 0, count); + for (var i = 0; i < count; i++) + { + if (mask[i] != 0) + { + result[i] = 1; + return result; + } + } + return result; + } + + public static int LastActiveElement(byte[] v) + { + for (var i = v.Length - 1; i >= 0; i--) + { + if (v[i] != 0) + { + return i; + } + } + return -1; + } + + public static int LastActiveElement(ushort[] v) + { + for (var i = v.Length - 1; i >= 0; i--) + { + if (v[i] != 0) + { + return i; + } + } + return -1; + } + + public static int LastActiveElement(uint[] v) + { + for (var i = v.Length - 1; i >= 0; i--) + { + if (v[i] != 0) + { + return i; + } + } + return -1; + } + + public static int LastActiveElement(ulong[] v) + { + for (var i = v.Length - 1; i >= 0; i--) + { + if (v[i] != 0) + { + return i; + } + } + return -1; + } + + public static byte[] CreateMaskForNextActiveElement(byte[] mask, byte[] srcMask) + { + var count = srcMask.Length; + var result = new byte[count]; + for (var i = LastActiveElement(srcMask) + 1; i < count; i++) + { + if (mask[i] != 0) + { + result[i] = 1; + return result; + } + } + return result; + } + + public static ushort[] CreateMaskForNextActiveElement(ushort[] mask, ushort[] srcMask) + { + var count = srcMask.Length; + var result = new ushort[count]; + for (var i = LastActiveElement(srcMask) + 1; i < count; i++) + { + if (mask[i] != 0) + { + result[i] = 1; + return result; + } + } + return result; + } + + public static uint[] CreateMaskForNextActiveElement(uint[] mask, uint[] srcMask) + { + var count = srcMask.Length; + var result = new uint[count]; + for (var i = LastActiveElement(srcMask) + 1; i < count; i++) + { + if (mask[i] != 0) + { + result[i] = 1; + return result; + } + } + return result; + } + + public static ulong[] CreateMaskForNextActiveElement(ulong[] mask, ulong[] srcMask) + { + var count = srcMask.Length; + var result = new ulong[count]; + for (var i = LastActiveElement(srcMask) + 1; i < count; i++) + { + if (mask[i] != 0) + { + result[i] = 1; + return result; + } + } + return result; + } + public static sbyte CountLeadingSignBits(sbyte op1) { return (sbyte)(CountLeadingZeroBits((sbyte)((ulong)op1 ^ ((ulong)op1 >> 1))) - 1);