Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[mono] Enable more HardwareIntrinsics tests and fix more amd64 intrinsics bugs #54127

Merged
merged 13 commits into from
Jun 17, 2021
Merged

[mono] Enable more HardwareIntrinsics tests and fix more amd64 intrinsics bugs #54127

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
287aca1
[mono] Enable more HardwareIntrinsics tests
imhameed Jun 13, 2021
854d632
Fix OP_MULX_HL{32,64}: the low-word address is sometimes an integer p…
imhameed Jun 14, 2021
796ff9c
Implement immediate unrolling for Ssse3.AlignRight
imhameed Jun 14, 2021
8cd4601
Implement the 3-argument overload of Bmi1.BitFieldExtract
imhameed Jun 14, 2021
9bf11e8
Implement immediate unrolling for Aes.KeygenAssist
imhameed Jun 14, 2021
7aa00cb
Disable StoreNonTemporal tests
imhameed Jun 14, 2021
2d7d9cd
Implement immediate unrolling for Sse41.MultipleSumAbsoluteDifferences
imhameed Jun 14, 2021
a6ceb01
Mask vector selection index in `OP_XEXTRACT_*` and `OP_XINSERT_*`
imhameed Jun 14, 2021
ae0d93c
Copy the upper lanes over to the destination in `OP_SSE_CMPSS` and `O…
imhameed Jun 14, 2021
5ddcf22
Disable JIT/HardwareIntrinsics/X86/Sse42.X64/Crc32_*
imhameed Jun 14, 2021
20f53f8
Implement immediate unrolling for `Pclmulqdq.CarrylessMultiply`
imhameed Jun 14, 2021
9b49167
Enable pclmul, aes when compiling tests. Insert `unreachable` instruc…
imhameed Jun 15, 2021
4dfc5df
emit_invalid_operation is unused
imhameed Jun 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions src/mono/mono/mini/llvm-intrinsics.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,15 +255,15 @@ INTRINS(AESNI_AESENCLAST, x86_aesni_aesenclast, X86)
INTRINS(AESNI_AESIMC, x86_aesni_aesimc, X86)
#if LLVM_API_VERSION >= 800
// these intrinsics were renamed in LLVM 8
INTRINS_OVR(SSE_SADD_SATI8, sadd_sat, Generic, sse_i1_t)
INTRINS_OVR(SSE_UADD_SATI8, uadd_sat, Generic, sse_i1_t)
INTRINS_OVR(SSE_SADD_SATI16, sadd_sat, Generic, sse_i1_t)
INTRINS_OVR(SSE_UADD_SATI16, uadd_sat, Generic, sse_i1_t)

INTRINS_OVR(SSE_SSUB_SATI8, ssub_sat, Generic, sse_i2_t)
INTRINS_OVR(SSE_USUB_SATI8, usub_sat, Generic, sse_i2_t)
INTRINS_OVR(SSE_SSUB_SATI16, ssub_sat, Generic, sse_i2_t)
INTRINS_OVR(SSE_USUB_SATI16, usub_sat, Generic, sse_i2_t)
INTRINS_OVR(SSE_SADD_SATI8, sadd_sat, Generic, v128_i1_t)
INTRINS_OVR(SSE_UADD_SATI8, uadd_sat, Generic, v128_i1_t)
INTRINS_OVR(SSE_SADD_SATI16, sadd_sat, Generic, v128_i2_t)
INTRINS_OVR(SSE_UADD_SATI16, uadd_sat, Generic, v128_i2_t)

INTRINS_OVR(SSE_SSUB_SATI8, ssub_sat, Generic, v128_i1_t)
INTRINS_OVR(SSE_USUB_SATI8, usub_sat, Generic, v128_i1_t)
INTRINS_OVR(SSE_SSUB_SATI16, ssub_sat, Generic, v128_i2_t)
INTRINS_OVR(SSE_USUB_SATI16, usub_sat, Generic, v128_i2_t)
#else
INTRINS(SSE_SADD_SATI8, x86_sse2_padds_b, X86)
INTRINS(SSE_UADD_SATI8, x86_sse2_paddus_b, X86)
Expand Down
229 changes: 129 additions & 100 deletions src/mono/mono/mini/mini-llvm.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5022,6 +5022,13 @@ immediate_unroll_commit_default (ImmediateUnrollCtx *ictx, LLVMValueRef value)
LLVMAddIncoming (ictx->phi, &value, &ictx->default_case, 1);
}

static void
immediate_unroll_unreachable_default (ImmediateUnrollCtx *ictx)
{
immediate_unroll_default (ictx);
LLVMBuildUnreachable (ictx->ctx->builder);
}

static LLVMValueRef
immediate_unroll_end (ImmediateUnrollCtx *ictx, LLVMBasicBlockRef *continuation)
{
Expand Down Expand Up @@ -7409,7 +7416,9 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb)
LLVMTypeRef lhs_t = LLVMTypeOf (lhs);
int vec_width = mono_llvm_get_prim_size_bits (lhs_t);
int elem_width = mono_llvm_get_prim_size_bits (elt_t);
LLVMTypeRef ret_t = LLVMVectorType (elt_t, vec_width / elem_width);
int elements = vec_width / elem_width;
element_ix = LLVMBuildAnd (builder, element_ix, const_int32 (elements - 1), "extract");
LLVMTypeRef ret_t = LLVMVectorType (elt_t, elements);
LLVMValueRef src = LLVMBuildBitCast (builder, lhs, ret_t, "extract");
LLVMValueRef result = LLVMBuildExtractElement (builder, src, element_ix, "extract");
if (zext)
Expand All @@ -7419,6 +7428,23 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb)
values [ins->dreg] = result;
break;
}
case OP_XINSERT_I1:
case OP_XINSERT_I2:
case OP_XINSERT_I4:
case OP_XINSERT_I8:
case OP_XINSERT_R4:
case OP_XINSERT_R8: {
MonoTypeEnum primty = inst_c1_type (ins);
LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass);
LLVMTypeRef elem_t = LLVMGetElementType (ret_t);
int elements = LLVMGetVectorSize (ret_t);
LLVMValueRef element_ix = LLVMBuildAnd (builder, arg3, const_int32 (elements - 1), "xinsert");
LLVMValueRef vec = convert (ctx, lhs, ret_t);
LLVMValueRef val = convert_full (ctx, rhs, elem_t, primitive_type_is_unsigned (primty));
LLVMValueRef result = LLVMBuildInsertElement (builder, vec, val, element_ix, "xinsert");
values [ins->dreg] = result;
break;
}
case OP_EXPAND_I1:
case OP_EXPAND_I2:
case OP_EXPAND_I4:
Expand Down Expand Up @@ -7785,46 +7811,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb)
values [ins->dreg] = LLVMBuildSExt (builder, pcmp, retType, "");
break;
}
case OP_XINSERT_I2: {
LLVMBasicBlockRef bbs [64];
LLVMValueRef switch_ins;
LLVMValueRef vector = lhs;
LLVMValueRef value = rhs;
LLVMValueRef index = values [ins->sreg3];
LLVMValueRef phi_values [64];
int nelems = LLVMGetVectorSize (LLVMTypeOf (lhs));
int i;

/*
* Many SIMD opcodes require an immediate operand, but can be called with a non-immediate.
* To handle these cases, generate a switch statement with one case for all possible
* values of the immediate.
* switch (index) {
* case i:
* res = <op> (val, i)
* break;
* }
*/
g_assert (nelems <= 64);
for (i = 0; i < nelems; ++i)
bbs [i] = gen_bb (ctx, "XINSERT_CASE_BB");
cbb = gen_bb (ctx, "XINSERT_COND_BB");

switch_ins = LLVMBuildSwitch (builder, LLVMBuildAnd (builder, index, const_int32 (0xf), ""), bbs [0], 0);
for (i = 0; i < nelems; ++i) {
LLVMAddCase (switch_ins, LLVMConstInt (LLVMInt32Type (), i, FALSE), bbs [i]);
LLVMPositionBuilderAtEnd (builder, bbs [i]);
phi_values [i] = LLVMBuildInsertElement (builder, vector, convert (ctx, value, LLVMGetElementType (LLVMTypeOf (vector))), LLVMConstInt (LLVMInt32Type (), i, FALSE), "");
LLVMBuildBr (builder, cbb);
}

LLVMPositionBuilderAtEnd (builder, cbb);
values [ins->dreg] = LLVMBuildPhi (builder, LLVMTypeOf (phi_values [0]), "");
LLVMAddIncoming (values [ins->dreg], phi_values, bbs, nelems);

ctx->bblocks [bb->block_num].end_bblock = cbb;
break;
}
case OP_CVTDQ2PS: {
LLVMValueRef i4 = LLVMBuildBitCast (builder, lhs, sse_i4_t, "");
values [ins->dreg] = LLVMBuildSIToFP (builder, i4, sse_r4_t, dname);
Expand Down Expand Up @@ -8443,7 +8429,14 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb)
case OP_SSE2_CMPSD: id = INTRINS_SSE_CMPSD; break;
default: g_assert_not_reached (); break;
}
values [ins->dreg] = call_intrins (ctx, id, args, "");
int elements = LLVMGetVectorSize (LLVMTypeOf (lhs));
int mask_values [MAX_VECTOR_ELEMS] = { 0 };
for (int i = 1; i < elements; ++i) {
mask_values [i] = elements + i;
}
LLVMValueRef result = call_intrins (ctx, id, args, "");
result = LLVMBuildShuffleVector (builder, result, lhs, create_const_vector_i32 (mask_values, elements), "");
values [ins->dreg] = result;
break;
}
case OP_SSE_COMISS: {
Expand Down Expand Up @@ -8914,16 +8907,37 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb)
values [ins->dreg] = convert (ctx, abs, typ);
break;
}

case OP_SSSE3_ALIGNR: {
LLVMValueRef mask_values [16];
for (int i = 0; i < 16; i++)
mask_values [i] = LLVMConstInt (LLVMInt32Type (), i + ins->inst_c0, FALSE);
LLVMValueRef shuffled = LLVMBuildShuffleVector (builder,
convert (ctx, rhs, sse_i1_t),
convert (ctx, lhs, sse_i1_t),
LLVMConstVector (mask_values, 16), "");
values [ins->dreg] = convert (ctx, shuffled, type_to_sse_type (ins->inst_c1));
LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass);
LLVMValueRef zero = LLVMConstNull (v128_i1_t);
LLVMValueRef hivec = convert (ctx, lhs, v128_i1_t);
LLVMValueRef lovec = convert (ctx, rhs, v128_i1_t);
LLVMValueRef rshift_amount = convert (ctx, arg3, i1_t);
ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, 32, rshift_amount, v128_i1_t, "ssse3_alignr");
LLVMValueRef mask_values [16]; // 128-bit vector, 8-bit elements, 16 total elements
int i = 0;
while (immediate_unroll_next (&ictx, &i)) {
LLVMValueRef hi = NULL;
LLVMValueRef lo = NULL;
if (i <= 16) {
for (int j = 0; j < 16; j++)
mask_values [j] = const_int32 (i + j);
lo = lovec;
hi = hivec;
} else {
for (int j = 0; j < 16; j++)
mask_values [j] = const_int32 (i + j - 16);
lo = hivec;
hi = zero;
}
LLVMValueRef shuffled = LLVMBuildShuffleVector (builder, lo, hi, LLVMConstVector (mask_values, 16), "ssse3_alignr");
immediate_unroll_commit (&ictx, i, shuffled);
}
immediate_unroll_default (&ictx);
immediate_unroll_commit_default (&ictx, zero);
LLVMValueRef result = immediate_unroll_end (&ictx, &cbb);
values [ins->dreg] = convert (ctx, result, ret_t);
break;
}

Expand Down Expand Up @@ -8989,30 +9003,40 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb)
break;
}

case OP_SSE41_MPSADBW_IMM: {
LLVMValueRef args [3];
args [0] = LLVMBuildBitCast (ctx->builder, lhs, sse_i1_t, "");
args [1] = LLVMBuildBitCast (ctx->builder, rhs, sse_i1_t, "");
args [2] = LLVMConstInt (LLVMInt8Type (), ins->inst_c0, FALSE);
values [ins->dreg] = call_intrins (ctx, INTRINS_SSE_MPSADBW, args, dname);
case OP_SSE41_MPSADBW: {
LLVMValueRef args [] = {
convert (ctx, lhs, sse_i1_t),
convert (ctx, rhs, sse_i1_t),
NULL,
};
LLVMValueRef ctl = convert (ctx, arg3, i1_t);
// Only 3 bits (bits 0-2) are used by mpsadbw and llvm.x86.sse41.mpsadbw
int used_bits = 0x7;
ctl = LLVMBuildAnd (builder, ctl, const_int8 (used_bits), "sse41_mpsadbw");
ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, used_bits + 1, ctl, v128_i2_t, "sse41_mpsadbw");
int i = 0;
while (immediate_unroll_next (&ictx, &i)) {
args [2] = const_int8 (i);
LLVMValueRef result = call_intrins (ctx, INTRINS_SSE_MPSADBW, args, "sse41_mpsadbw");
immediate_unroll_commit (&ictx, i, result);
}
immediate_unroll_unreachable_default (&ictx);
values [ins->dreg] = immediate_unroll_end (&ictx, &cbb);
break;
}

case OP_SSE41_INSERT: {
if (ins->inst_c1 == MONO_TYPE_R4) {
// special case for <float> overload
LLVMValueRef args [3];
args [0] = values [ins->sreg1];
args [1] = values [ins->sreg2];
args [2] = convert (ctx, values [ins->sreg3], LLVMInt8Type ());
values [ins->dreg] = call_intrins (ctx, INTRINS_SSE_INSERTPS, args, dname);
} else {
// other overloads are implemented with `insertelement`
values [ins->dreg] = LLVMBuildInsertElement (builder,
values [ins->sreg1],
convert (ctx, values [ins->sreg2], primitive_type_to_llvm_type (inst_c1_type (ins))),
convert (ctx, values [ins->sreg3], LLVMInt8Type ()), dname);
case OP_SSE41_INSERTPS: {
LLVMValueRef ctl = convert (ctx, arg3, i1_t);
LLVMValueRef args [] = { lhs, rhs, NULL };
ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, 256, ctl, v128_r4_t, "sse41_insertps");
int i = 0;
while (immediate_unroll_next (&ictx, &i)) {
args [2] = const_int8 (i);
LLVMValueRef result = call_intrins (ctx, INTRINS_SSE_INSERTPS, args, dname);
immediate_unroll_commit (&ictx, i, result);
}
immediate_unroll_unreachable_default (&ictx);
values [ins->dreg] = immediate_unroll_end (&ictx, &cbb);
break;
}

Expand Down Expand Up @@ -9157,15 +9181,37 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb)
break;
}

case OP_PCLMULQDQ_IMM: {
LLVMValueRef args [] = { lhs, rhs, LLVMConstInt (LLVMInt8Type (), ins->inst_c0, FALSE) };
values [ins->dreg] = call_intrins (ctx, INTRINS_PCLMULQDQ, args, "");
case OP_PCLMULQDQ: {
LLVMValueRef args [] = { lhs, rhs, NULL };
LLVMValueRef ctl = convert (ctx, arg3, i1_t);
// Only bits 0 and 4 of the immediate operand are used by PCLMULQDQ.
ctl = LLVMBuildAnd (builder, ctl, const_int8 (0x11), "pclmulqdq");
ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, 1 << 2, ctl, v128_i8_t, "pclmulqdq");
int i = 0;
while (immediate_unroll_next (&ictx, &i)) {
int imm = ((i & 0x2) << 3) | (i & 0x1);
args [2] = const_int8 (imm);
LLVMValueRef result = call_intrins (ctx, INTRINS_PCLMULQDQ, args, "pclmulqdq");
immediate_unroll_commit (&ictx, imm, result);
}
immediate_unroll_unreachable_default (&ictx);
values [ins->dreg] = immediate_unroll_end (&ictx, &cbb);
break;
}

case OP_AES_KEYGEN_IMM: {
LLVMValueRef args [] = { lhs, LLVMConstInt (LLVMInt8Type (), ins->inst_c0, FALSE) };
values [ins->dreg] = call_intrins (ctx, INTRINS_AESNI_AESKEYGENASSIST, args, "");
case OP_AES_KEYGENASSIST: {
LLVMValueRef roundconstant = convert (ctx, rhs, i1_t);
LLVMValueRef args [] = { convert (ctx, lhs, v128_i8_t), NULL };
ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, 256, roundconstant, v128_i8_t, "aes_keygenassist");
int i = 0;
while (immediate_unroll_next (&ictx, &i)) {
args [1] = const_int8 (i);
LLVMValueRef result = call_intrins (ctx, INTRINS_AESNI_AESKEYGENASSIST, args, "aes_keygenassist");
immediate_unroll_commit (&ictx, i, result);
}
immediate_unroll_unreachable_default (&ictx);
LLVMValueRef result = immediate_unroll_end (&ictx, &cbb);
values [ins->dreg] = convert (ctx, result, v128_i1_t);
break;
}
#endif
Expand Down Expand Up @@ -9275,12 +9321,12 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb)
values [ins->dreg] = call_intrins (ctx, ins->opcode == OP_CTTZ32 ? INTRINS_CTTZ_I32 : INTRINS_CTTZ_I64, args, "");
break;
}
case OP_BEXTR32:
case OP_BEXTR64: {
case OP_BMI1_BEXTR32:
case OP_BMI1_BEXTR64: {
LLVMValueRef args [2];
args [0] = lhs;
args [1] = convert (ctx, rhs, ins->opcode == OP_BEXTR32 ? LLVMInt32Type () : LLVMInt64Type ()); // cast ushort to u32/u64
values [ins->dreg] = call_intrins (ctx, ins->opcode == OP_BEXTR32 ? INTRINS_BEXTR_I32 : INTRINS_BEXTR_I64, args, "");
args [1] = convert (ctx, rhs, ins->opcode == OP_BMI1_BEXTR32 ? i4_t : i8_t); // cast ushort to u32/u64
values [ins->dreg] = call_intrins (ctx, ins->opcode == OP_BMI1_BEXTR32 ? INTRINS_BEXTR_I32 : INTRINS_BEXTR_I64, args, "");
break;
}
case OP_BZHI32:
Expand All @@ -9301,8 +9347,9 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb)
LLVMValueRef rx = LLVMBuildZExt (ctx->builder, rhs, LLVMInt128Type (), "");
LLVMValueRef mulx = LLVMBuildMul (ctx->builder, lx, rx, "");
if (!only_high) {
LLVMValueRef addr = convert (ctx, arg3, LLVMPointerType (is_64 ? i8_t : i4_t, 0));
LLVMValueRef lowx = LLVMBuildTrunc (ctx->builder, mulx, is_64 ? LLVMInt64Type () : LLVMInt32Type (), "");
LLVMBuildStore (ctx->builder, lowx, values [ins->sreg3]);
LLVMBuildStore (ctx->builder, lowx, addr);
}
LLVMValueRef shift = LLVMConstInt (LLVMInt128Type (), is_64 ? 64 : 32, FALSE);
LLVMValueRef highx = LLVMBuildLShr (ctx->builder, mulx, shift, "");
Expand Down Expand Up @@ -9419,24 +9466,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb)
values [ins->dreg] = result;
break;
}
case OP_XINSERT_I1:
case OP_XINSERT_I2:
case OP_XINSERT_I4:
case OP_XINSERT_I8:
case OP_XINSERT_R4:
case OP_XINSERT_R8: {
/* TODO: Scalar types smaller than i32 seem to be
* normalized to i32 via zero or sign extension.
* Is this still necessary?
*/
LLVMTypeRef t = LLVMTypeOf (lhs);
LLVMTypeRef elem_t = LLVMGetElementType (t);
MonoTypeEnum primty = inst_c1_type (ins);
LLVMValueRef val = convert_full (ctx, rhs, elem_t, primitive_type_is_unsigned (primty));
LLVMValueRef result = LLVMBuildInsertElement (builder, lhs, val, arg3, "xinsert");
values [ins->dreg] = result;
break;
}
case OP_XCOMPARE_FP_SCALAR:
case OP_XCOMPARE_FP: {
g_assert (LLVMTypeOf (lhs) == LLVMTypeOf (rhs));
Expand Down
20 changes: 12 additions & 8 deletions src/mono/mono/mini/mini-ops.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1097,7 +1097,7 @@ MINI_OP3(OP_SSSE3_ALIGNR, "ssse3_alignr", XREG, XREG, XREG, IREG)
/* sse 4.1 */
MINI_OP(OP_SSE41_ROUNDP, "roundp", XREG, XREG, NONE) // packed, inst_c0 - mode, inst_c1 - r4 or r8
MINI_OP(OP_SSE41_ROUNDS, "sse41_rounds", XREG, XREG, XREG) // scalar, inst_c0 - mode, inst_c1 - r4 or r8
MINI_OP3(OP_SSE41_INSERT, "sse41_insert", XREG, XREG, XREG, IREG)
MINI_OP3(OP_SSE41_INSERTPS, "sse41_insertps", XREG, XREG, XREG, IREG)
MINI_OP3(OP_SSE41_BLENDV, "sse41_blendv", XREG, XREG, XREG, XREG)
MINI_OP3(OP_SSE41_BLEND, "sse41_blend", XREG, XREG, XREG, IREG)
MINI_OP(OP_SSE41_LOADANT, "sse41_loadant", XREG, XREG, NONE)
Expand All @@ -1106,13 +1106,13 @@ MINI_OP(OP_SSE41_MULLO, "sse41_mullo", XREG, XREG, XREG)
MINI_OP(OP_SSE_CVTII, "sse_cvtii", XREG, XREG, NONE)
MINI_OP3(OP_SSE41_DPPS, "sse41_dpps", XREG, XREG, XREG, IREG)
MINI_OP3(OP_SSE41_DPPD, "sse41_dppd", XREG, XREG, XREG, IREG)
MINI_OP(OP_SSE41_MPSADBW_IMM, "sse_mpsadbw", XREG, XREG, XREG)
MINI_OP3(OP_SSE41_MPSADBW, "sse41_mpsadbw", XREG, XREG, XREG, IREG)

/* pclmulqdq */
MINI_OP(OP_PCLMULQDQ_IMM, "pclmulqdq", XREG, XREG, XREG)
MINI_OP3(OP_PCLMULQDQ, "pclmulqdq", XREG, XREG, XREG, IREG)

/* aes */
MINI_OP(OP_AES_KEYGEN_IMM, "aes_keygen", XREG, XREG, NONE)
MINI_OP(OP_AES_KEYGENASSIST, "aes_keygen", XREG, XREG, IREG)

/* sse 4.2 */
MINI_OP(OP_SSE42_CRC32, "sse42_crc32", IREG, IREG, IREG)
Expand All @@ -1123,8 +1123,8 @@ MINI_OP(OP_SSE42_PTESTZ, "sse42_ptestc", IREG, XREG, XREG)
/* Count trailing zeroes, return 32/64 if the input is 0 */
MINI_OP(OP_CTTZ32, "cttz32", IREG, IREG, NONE)
MINI_OP(OP_CTTZ64, "cttz64", LREG, LREG, NONE)
MINI_OP(OP_BEXTR32, "bextr32", IREG, IREG, IREG)
MINI_OP(OP_BEXTR64, "bextr64", LREG, LREG, LREG)
MINI_OP(OP_BMI1_BEXTR32, "bmi1_bextr32", IREG, IREG, IREG)
MINI_OP(OP_BMI1_BEXTR64, "bmi1_bextr64", LREG, LREG, LREG)

/* Intel BMI2 */
MINI_OP(OP_BZHI32, "bzhi32", IREG, IREG, IREG)
Expand Down Expand Up @@ -1576,8 +1576,12 @@ MINI_OP(OP_XEXTRACT_I8, "xextract_i8", LREG, XREG, IREG)
MINI_OP(OP_XEXTRACT_R4, "xextract_r4", FREG, XREG, IREG)
MINI_OP(OP_XEXTRACT_R8, "xextract_r8", FREG, XREG, IREG)

/* Insert element into a vector */
/* sreg1 is the vector, sreg2 is the value, sreg3 is the index */
/* Insert an element into a vector with a variable lane index.
* sreg1 is the vector, sreg2 is the value, sreg3 is the index.
* inst_c1 is a MonoTypeEnum representing the element type, used to control
* sign/zero extension if conversion of sreg2 happens to be necessary.
* klass must be set to the MonoClass representing the return type of the op.
*/
MINI_OP3(OP_XINSERT_I1, "xinsert_i1", XREG, XREG, IREG, IREG)
MINI_OP3(OP_XINSERT_I2, "xinsert_i2", XREG, XREG, IREG, IREG)
MINI_OP3(OP_XINSERT_I4, "xinsert_i4", XREG, XREG, IREG, IREG)
Expand Down
Loading