From 9cfd11d4f09755d1695029aed300579f62a0a660 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 1 Mar 2023 22:04:39 +0100 Subject: [PATCH] JIT: Unify arm64 and x64 GT_SELECT handling (#82610) This unifies GT_SELECT/GT_SELECTCC handling between arm64 and x64. The arm64 backend no longer uses containment for compare chains; instead, there is a new GT_CCMP node that both produces and consumes flags, and lowering can lower GT_AND(op, relop) down to this node. --- src/coreclr/jit/codegen.h | 4 +- src/coreclr/jit/codegenarm64.cpp | 223 ++++--------------------- src/coreclr/jit/codegenarmarch.cpp | 8 + src/coreclr/jit/gentree.cpp | 30 +++- src/coreclr/jit/gentree.h | 57 ++++++- src/coreclr/jit/gtlist.h | 8 +- src/coreclr/jit/gtstructs.h | 3 + src/coreclr/jit/instr.h | 23 --- src/coreclr/jit/lower.cpp | 100 ++++++++++-- src/coreclr/jit/lower.h | 11 +- src/coreclr/jit/lowerarmarch.cpp | 251 +++++++++++------------------ src/coreclr/jit/lsraarm64.cpp | 5 + src/coreclr/jit/lsraarmarch.cpp | 9 +- src/coreclr/jit/lsrabuild.cpp | 4 +- 14 files changed, 331 insertions(+), 405 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 4dc279b589dd1..04352f6e1e30f 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -883,8 +883,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void genCkfinite(GenTree* treeNode); void genCodeForCompare(GenTreeOp* tree); #ifdef TARGET_ARM64 - void genCodeForConditionalCompare(GenTreeOp* tree, GenCondition prevCond); - void genCodeForContainedCompareChain(GenTree* tree, bool* inchain, GenCondition* prevCond); + void genCodeForCCMP(GenTreeCCMP* ccmp); #endif void genCodeForSelect(GenTreeOp* select); void genIntrinsic(GenTreeIntrinsic* treeNode); @@ -1559,7 +1558,6 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #endif // TARGET_XARCH #if defined(TARGET_ARM64) - static insCflags InsCflagsForCcmp(GenCondition cond); static insCond JumpKindToInsCond(emitJumpKind condition); #elif defined(TARGET_XARCH) static instruction JumpKindToCmov(emitJumpKind condition); diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index dae6ea2105208..c48ea9aba89f1 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -2696,35 +2696,6 @@ void CodeGen::genCodeForBinary(GenTreeOp* tree) return; } - if (tree->isContainedCompareChainSegment(op2)) - { - GenCondition cond; - bool chain = false; - - JITDUMP("Generating compare chain:\n"); - if (op1->isContained()) - { - // Generate Op1 into flags. - genCodeForContainedCompareChain(op1, &chain, &cond); - assert(chain); - } - else - { - // Op1 is not contained, move it from a register into flags. - emit->emitIns_R_I(INS_cmp, emitActualTypeSize(op1), op1->GetRegNum(), 0); - cond = GenCondition::NE; - chain = true; - } - // Gen Op2 into flags. - genCodeForContainedCompareChain(op2, &chain, &cond); - assert(chain); - - // Move the result from flags into a register. - inst_SETCC(cond, tree->TypeGet(), targetReg); - genProduceReg(tree); - return; - } - instruction ins = genGetInsForOper(tree->OperGet(), targetType); if ((tree->gtFlags & GTF_SET_FLAGS) != 0) @@ -4600,108 +4571,36 @@ void CodeGen::genCodeForCompare(GenTreeOp* tree) // tree - a compare node (GT_EQ etc) // cond - the condition of the previous generated compare. // -void CodeGen::genCodeForConditionalCompare(GenTreeOp* tree, GenCondition prevCond) +void CodeGen::genCodeForCCMP(GenTreeCCMP* ccmp) { emitter* emit = GetEmitter(); - GenTree* op1 = tree->gtGetOp1(); - GenTree* op2 = tree->gtGetOp2(); - var_types op1Type = genActualType(op1->TypeGet()); - var_types op2Type = genActualType(op2->TypeGet()); - emitAttr cmpSize = EA_ATTR(genTypeSize(op1Type)); - regNumber targetReg = tree->GetRegNum(); - regNumber srcReg1 = op1->GetRegNum(); + genConsumeOperands(ccmp); + GenTree* op1 = ccmp->gtGetOp1(); + GenTree* op2 = ccmp->gtGetOp2(); + var_types op1Type = genActualType(op1->TypeGet()); + var_types op2Type = genActualType(op2->TypeGet()); + emitAttr cmpSize = emitActualTypeSize(op1Type); + regNumber srcReg1 = op1->GetRegNum(); // No float support or swapping op1 and op2 to generate cmp reg, imm. assert(!varTypeIsFloating(op2Type)); assert(!op1->isContainedIntOrIImmed()); - // Should only be called on contained nodes. - assert(targetReg == REG_NA); - - // Should not be called for test conditionals (Arm64 does not have a ctst). - assert(tree->OperIsCmpCompare()); - // For the ccmp flags, invert the condition of the compare. - insCflags cflags = InsCflagsForCcmp(GenCondition::FromRelop(tree)); - // For the condition, use the previous compare. - const GenConditionDesc& prevDesc = GenConditionDesc::Get(prevCond); - insCond prevInsCond = JumpKindToInsCond(prevDesc.jumpKind1); + const GenConditionDesc& condDesc = GenConditionDesc::Get(ccmp->gtCondition); + insCond insCond = JumpKindToInsCond(condDesc.jumpKind1); if (op2->isContainedIntOrIImmed()) { GenTreeIntConCommon* intConst = op2->AsIntConCommon(); - emit->emitIns_R_I_FLAGS_COND(INS_ccmp, cmpSize, srcReg1, (int)intConst->IconValue(), cflags, prevInsCond); + emit->emitIns_R_I_FLAGS_COND(INS_ccmp, cmpSize, srcReg1, (int)intConst->IconValue(), ccmp->gtFlagsVal, insCond); } else { regNumber srcReg2 = op2->GetRegNum(); - emit->emitIns_R_R_FLAGS_COND(INS_ccmp, cmpSize, srcReg1, srcReg2, cflags, prevInsCond); - } -} - -//------------------------------------------------------------------------ -// genCodeForContainedCompareChain: Produce code for a chain of conditional compares. -// -// Only generates for contained nodes. Nodes that are not contained are assumed to be -// generated as part of standard tree generation. -// -// Arguments: -// tree - the node. Either a compare or a tree of compares connected by ANDs. -// inChain - whether a contained chain is in progress. -// prevCond - If a chain is in progress, the condition of the previous compare. -// Return: -// The last compare node generated. -// -void CodeGen::genCodeForContainedCompareChain(GenTree* tree, bool* inChain, GenCondition* prevCond) -{ - assert(tree->isContained()); - - if (tree->OperIs(GT_AND)) - { - GenTree* op1 = tree->gtGetOp1(); - GenTree* op2 = tree->gtGetOp2(); - - assert(op2->isContained()); - - // If Op1 is contained, generate into flags. Otherwise, move the result into flags. - if (op1->isContained()) - { - genCodeForContainedCompareChain(op1, inChain, prevCond); - assert(*inChain); - } - else - { - emitter* emit = GetEmitter(); - emit->emitIns_R_I(INS_cmp, emitActualTypeSize(op1), op1->GetRegNum(), 0); - *prevCond = GenCondition::NE; - *inChain = true; - } - - // Generate Op2 based on Op1. - genCodeForContainedCompareChain(op2, inChain, prevCond); - assert(*inChain); - } - else - { - assert(tree->OperIsCmpCompare()); - - // Generate the compare, putting the result in the flags register. - if (!*inChain) - { - // First item in a chain. Use a standard compare. - genCodeForCompare(tree->AsOp()); - } - else - { - // Within the chain. Use a conditional compare (which is - // dependent on the previous emitted compare). - genCodeForConditionalCompare(tree->AsOp(), *prevCond); - } - - *inChain = true; - *prevCond = GenCondition::FromRelop(tree); + emit->emitIns_R_R_FLAGS_COND(INS_ccmp, cmpSize, srcReg1, srcReg2, ccmp->gtFlagsVal, insCond); } } @@ -4713,45 +4612,37 @@ void CodeGen::genCodeForContainedCompareChain(GenTree* tree, bool* inChain, GenC // void CodeGen::genCodeForSelect(GenTreeOp* tree) { - assert(tree->OperIs(GT_SELECT)); - GenTreeConditional* select = tree->AsConditional(); - emitter* emit = GetEmitter(); + assert(tree->OperIs(GT_SELECT, GT_SELECTCC)); + GenTree* opcond = nullptr; + if (tree->OperIs(GT_SELECT)) + { + opcond = tree->AsConditional()->gtCond; + genConsumeRegs(opcond); + } - GenTree* opcond = select->gtCond; - GenTree* op1 = select->gtOp1; - GenTree* op2 = select->gtOp2; - var_types op1Type = genActualType(op1->TypeGet()); - var_types op2Type = genActualType(op2->TypeGet()); - emitAttr attr = emitActualTypeSize(select->TypeGet()); + emitter* emit = GetEmitter(); + + GenTree* op1 = tree->gtOp1; + GenTree* op2 = tree->gtOp2; + var_types op1Type = genActualType(op1); + var_types op2Type = genActualType(op2); + emitAttr attr = emitActualTypeSize(tree); assert(!op1->isUsedFromMemory()); assert(genTypeSize(op1Type) == genTypeSize(op2Type)); - GenCondition prevCond; - genConsumeRegs(opcond); - if (opcond->isContained()) + GenCondition cond; + + if (opcond != nullptr) { - // Generate the contained condition. - if (opcond->OperIsCompare()) - { - genCodeForCompare(opcond->AsOp()); - prevCond = GenCondition::FromRelop(opcond); - } - else - { - // Condition is a compare chain. Try to contain it. - assert(opcond->OperIs(GT_AND)); - bool chain = false; - JITDUMP("Generating compare chain:\n"); - genCodeForContainedCompareChain(opcond, &chain, &prevCond); - assert(chain); - } + // Condition has been generated into a register - move it into flags. + emit->emitIns_R_I(INS_cmp, emitActualTypeSize(opcond), opcond->GetRegNum(), 0); + cond = GenCondition::NE; } else { - // Condition has been generated into a register - move it into flags. - emit->emitIns_R_I(INS_cmp, emitActualTypeSize(opcond), opcond->GetRegNum(), 0); - prevCond = GenCondition::NE; + assert(tree->OperIs(GT_SELECTCC)); + cond = tree->AsOpCC()->gtCondition; } assert(!op1->isContained() || op1->IsIntegralConst(0)); @@ -4760,7 +4651,7 @@ void CodeGen::genCodeForSelect(GenTreeOp* tree) regNumber targetReg = tree->GetRegNum(); regNumber srcReg1 = op1->IsIntegralConst(0) ? REG_ZR : genConsumeReg(op1); regNumber srcReg2 = op2->IsIntegralConst(0) ? REG_ZR : genConsumeReg(op2); - const GenConditionDesc& prevDesc = GenConditionDesc::Get(prevCond); + const GenConditionDesc& prevDesc = GenConditionDesc::Get(cond); emit->emitIns_R_R_R_COND(INS_csel, attr, targetReg, srcReg1, srcReg2, JumpKindToInsCond(prevDesc.jumpKind1)); @@ -10382,50 +10273,6 @@ void CodeGen::genCodeForCond(GenTreeOp* tree) genProduceReg(tree); } -//------------------------------------------------------------------------ -// InsCflagsForCcmp: Get the Cflags for a required for a CCMP instruction. -// -// Consider: -// cmp w, x -// ccmp y, z, A, COND -// This is: compare w and x, if this matches condition COND, then compare y and z. -// Otherwise set flags to A - this should match the case where cmp failed. -// Given COND, this function returns A. -// -// Arguments: -// cond - the GenCondition. -// -insCflags CodeGen::InsCflagsForCcmp(GenCondition cond) -{ - GenCondition inverted = GenCondition::Reverse(cond); - switch (inverted.GetCode()) - { - case GenCondition::EQ: - return INS_FLAGS_Z; - case GenCondition::NE: - return INS_FLAGS_NONE; - case GenCondition::SGE: - return INS_FLAGS_Z; - case GenCondition::SGT: - return INS_FLAGS_NONE; - case GenCondition::SLT: - return INS_FLAGS_NC; - case GenCondition::SLE: - return INS_FLAGS_NZC; - case GenCondition::UGE: - return INS_FLAGS_C; - case GenCondition::UGT: - return INS_FLAGS_C; - case GenCondition::ULT: - return INS_FLAGS_NONE; - case GenCondition::ULE: - return INS_FLAGS_Z; - default: - NO_WAY("unexpected condition type"); - return INS_FLAGS_NONE; - } -} - //------------------------------------------------------------------------ // JumpKindToInsCond: Convert a Jump Kind to a condition. // diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index c325e96e576f5..3bc0436a5544a 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -361,12 +361,20 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) case GT_SELECT: genCodeForSelect(treeNode->AsConditional()); break; + + case GT_SELECTCC: + genCodeForSelect(treeNode->AsOp()); + break; #endif #ifdef TARGET_ARM64 case GT_JCMP: genCodeForJumpCompare(treeNode->AsOp()); break; + + case GT_CCMP: + genCodeForCCMP(treeNode->AsCCMP()); + break; #endif // TARGET_ARM64 case GT_JCC: diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index abe3639c56802..bf8a6ef3b9268 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -313,6 +313,10 @@ void GenTree::InitNodeSize() static_assert_no_msg(sizeof(GenTreeLclFld) <= TREE_NODE_SZ_SMALL); static_assert_no_msg(sizeof(GenTreeCC) <= TREE_NODE_SZ_SMALL); static_assert_no_msg(sizeof(GenTreeOpCC) <= TREE_NODE_SZ_SMALL); +#ifdef TARGET_ARM64 + static_assert_no_msg(sizeof(GenTreeCCMP) <= TREE_NODE_SZ_SMALL); +#endif + static_assert_no_msg(sizeof(GenTreeConditional) <= TREE_NODE_SZ_SMALL); static_assert_no_msg(sizeof(GenTreeCast) <= TREE_NODE_SZ_LARGE); // *** large node static_assert_no_msg(sizeof(GenTreeBox) <= TREE_NODE_SZ_LARGE); // *** large node static_assert_no_msg(sizeof(GenTreeField) <= TREE_NODE_SZ_LARGE); // *** large node @@ -11317,6 +11321,17 @@ void Compiler::gtDispLclVarStructType(unsigned lclNum) } } +#if defined(DEBUG) && defined(TARGET_ARM64) +static const char* InsCflagsToString(insCflags flags) +{ + const static char* s_table[16] = {"0", "v", "c", "cv", "z", "zv", "zc", "zcv", + "n", "nv", "nc", "ncv", "nz", "nzv", "nzc", "nzcv"}; + unsigned index = (unsigned)flags; + assert((0 <= index) && (index < ArrLen(s_table))); + return s_table[index]; +} +#endif + //------------------------------------------------------------------------ // gtDispSsaName: Display the SSA use/def for a given local. // @@ -12162,6 +12177,13 @@ void Compiler::gtDispTree(GenTree* tree, { printf(" cond=%s", tree->AsOpCC()->gtCondition.Name()); } +#ifdef TARGET_ARM64 + else if (tree->OperIs(GT_CCMP)) + { + printf(" cond=%s flags=%s", tree->AsCCMP()->gtCondition.Name(), + InsCflagsToString(tree->AsCCMP()->gtFlagsVal)); + } +#endif gtDispCommonEndLine(tree); @@ -18837,7 +18859,13 @@ bool GenTree::SupportsSettingZeroFlag() } #endif #elif defined(TARGET_ARM64) - if (OperIs(GT_AND, GT_ADD, GT_SUB)) + if (OperIs(GT_AND)) + { + return true; + } + + // We do not support setting zero flag for madd/msub. + if (OperIs(GT_ADD, GT_SUB) && (!gtGetOp2()->OperIs(GT_MUL) || !gtGetOp2()->isContained())) { return true; } diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 9161226118cd9..d44e6ad138b5c 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -894,12 +894,6 @@ struct GenTree return isContained() && IsCnsIntOrI() && !isUsedFromSpillTemp(); } - // Node and its child in isolation form a contained compare chain. - bool isContainedCompareChainSegment(GenTree* child) const - { - return (OperIs(GT_AND) && child->isContained() && (child->OperIs(GT_AND) || child->OperIsCmpCompare())); - } - bool isContainedFltOrDblImmed() const { return isContained() && OperIs(GT_CNS_DBL); @@ -1693,7 +1687,15 @@ struct GenTree { #if !defined(TARGET_64BIT) if (OperIs(GT_ADD_HI, GT_SUB_HI)) + { + return true; + } +#endif +#if defined(TARGET_ARM64) + if (OperIs(GT_CCMP)) + { return true; + } #endif return OperIs(GT_JCC, GT_SETCC, GT_SELECTCC); } @@ -8537,7 +8539,7 @@ struct GenTreeCC final : public GenTree }; // Represents a node with two operands and a condition. -struct GenTreeOpCC final : public GenTreeOp +struct GenTreeOpCC : public GenTreeOp { GenCondition gtCondition; @@ -8554,6 +8556,47 @@ struct GenTreeOpCC final : public GenTreeOp #endif // DEBUGGABLE_GENTREE }; +#ifdef TARGET_ARM64 +enum insCflags : unsigned +{ + INS_FLAGS_NONE, + INS_FLAGS_V, + INS_FLAGS_C, + INS_FLAGS_CV, + + INS_FLAGS_Z, + INS_FLAGS_ZV, + INS_FLAGS_ZC, + INS_FLAGS_ZCV, + + INS_FLAGS_N, + INS_FLAGS_NV, + INS_FLAGS_NC, + INS_FLAGS_NCV, + + INS_FLAGS_NZ, + INS_FLAGS_NZV, + INS_FLAGS_NZC, + INS_FLAGS_NZCV, +}; + +struct GenTreeCCMP final : public GenTreeOpCC +{ + insCflags gtFlagsVal; + + GenTreeCCMP(var_types type, GenCondition condition, GenTree* op1, GenTree* op2, insCflags flagsVal) + : GenTreeOpCC(GT_CCMP, type, condition, op1, op2), gtFlagsVal(flagsVal) + { + } + +#if DEBUGGABLE_GENTREE + GenTreeCCMP() : GenTreeOpCC() + { + } +#endif // DEBUGGABLE_GENTREE +}; +#endif + //------------------------------------------------------------------------ // Deferred inline functions of GenTree -- these need the subtypes above to // be defined already. diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index eb734e7ec767f..13c5a70dfbc55 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -243,7 +243,13 @@ GTNODE(JCC , GenTreeCC ,0,GTK_LEAF|GTK_NOVALUE|DBK_NOTHIR) // Checks the condition flags and produces 1 if the condition specified by GenTreeCC::gtCondition is true and 0 otherwise. GTNODE(SETCC , GenTreeCC ,0,GTK_LEAF|DBK_NOTHIR) // Variant of SELECT that reuses flags computed by a previous node with the specified condition. -GTNODE(SELECTCC , GenTreeCC ,0,GTK_BINOP|DBK_NOTHIR) +GTNODE(SELECTCC , GenTreeOpCC ,0,GTK_BINOP|DBK_NOTHIR) +#ifdef TARGET_ARM64 +// The arm64 ccmp instruction. If the specified condition is true, compares two +// operands and sets the condition flags according to the result. Otherwise +// sets the condition flags to the specified immediate value. +GTNODE(CCMP , GenTreeCCMP ,0,GTK_BINOP|GTK_NOVALUE|DBK_NOTHIR) +#endif //----------------------------------------------------------------------------- diff --git a/src/coreclr/jit/gtstructs.h b/src/coreclr/jit/gtstructs.h index 052a6d09a8812..b64ff20f64e7a 100644 --- a/src/coreclr/jit/gtstructs.h +++ b/src/coreclr/jit/gtstructs.h @@ -113,6 +113,9 @@ GTSTRUCT_1(RuntimeLookup, GT_RUNTIMELOOKUP) GTSTRUCT_1(ArrAddr , GT_ARR_ADDR) GTSTRUCT_2(CC , GT_JCC, GT_SETCC) GTSTRUCT_1(OpCC , GT_SELECTCC) +#ifdef TARGET_ARM64 +GTSTRUCT_1(CCMP , GT_CCMP) +#endif #if defined(TARGET_X86) GTSTRUCT_1(MultiRegOp , GT_MUL_LONG) #elif defined (TARGET_ARM) diff --git a/src/coreclr/jit/instr.h b/src/coreclr/jit/instr.h index 180ad19ad3a96..bb5fc454ea439 100644 --- a/src/coreclr/jit/instr.h +++ b/src/coreclr/jit/instr.h @@ -269,29 +269,6 @@ enum insCond : unsigned INS_COND_LE, }; -enum insCflags : unsigned -{ - INS_FLAGS_NONE, - INS_FLAGS_V, - INS_FLAGS_C, - INS_FLAGS_CV, - - INS_FLAGS_Z, - INS_FLAGS_ZV, - INS_FLAGS_ZC, - INS_FLAGS_ZCV, - - INS_FLAGS_N, - INS_FLAGS_NV, - INS_FLAGS_NC, - INS_FLAGS_NCV, - - INS_FLAGS_NZ, - INS_FLAGS_NZV, - INS_FLAGS_NZC, - INS_FLAGS_NZCV, -}; - enum insBarrier : unsigned { INS_BARRIER_OSHLD = 1, diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index e21bb90c49221..81247ca991f77 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -234,6 +234,76 @@ bool Lowering::IsInvariantInRange(GenTree* node, GenTree* endExclusive, GenTree* return true; } +//------------------------------------------------------------------------ +// IsRangeInvariantInRange: Check if a range of nodes are invariant in the +// specified range. +// +// Arguments: +// rangeStart - The first node. +// rangeEnd - The last node. +// endExclusive - The exclusive end of the range to check invariance for. +// ignoreNode - A node to ignore interference checks with, for example +// because it will retain its relative order with 'node'. +// +// Returns: +// True if the range can be evaluated at any point between its current location +// and 'endExclusive' without giving a different result; otherwise false. +// +// Remarks: +// Note that the range is treated as a unit and no pairwise interference +// checks between nodes in the range are performed. +// +bool Lowering::IsRangeInvariantInRange(GenTree* rangeStart, + GenTree* rangeEnd, + GenTree* endExclusive, + GenTree* ignoreNode) const +{ + assert((rangeStart != nullptr) && (rangeEnd != nullptr)); + + if ((rangeEnd->gtNext == endExclusive) || + ((ignoreNode != nullptr) && (rangeEnd->gtNext == ignoreNode) && (rangeEnd->gtNext->gtNext == endExclusive))) + { + return true; + } + + if (rangeStart->OperConsumesFlags()) + { + return false; + } + + m_scratchSideEffects.Clear(); + GenTree* cur = rangeStart; + while (true) + { + m_scratchSideEffects.AddNode(comp, cur); + + if (cur == rangeEnd) + { + break; + } + + cur = cur->gtNext; + assert((cur != nullptr) && "Expected rangeStart to precede rangeEnd"); + } + + for (GenTree* cur = rangeEnd->gtNext; cur != endExclusive; cur = cur->gtNext) + { + assert((cur != nullptr) && "Expected first node to precede end node"); + if (cur == ignoreNode) + { + continue; + } + + const bool strict = true; + if (m_scratchSideEffects.InterferesWith(comp, cur, strict)) + { + return false; + } + } + + return true; +} + //------------------------------------------------------------------------ // IsSafeToContainMem: Checks for conflicts between childNode and parentNode, // and returns 'true' iff memory operand childNode can be contained in parentNode. @@ -2994,15 +3064,6 @@ GenTree* Lowering::OptimizeConstCompare(GenTree* cmp) GenTreeIntCon* op2 = cmp->gtGetOp2()->AsIntCon(); ssize_t op2Value = op2->IconValue(); -#ifdef TARGET_ARM64 - // Do not optimise further if op1 has a contained chain. - if (op1->OperIs(GT_AND) && - (op1->isContainedCompareChainSegment(op1->gtGetOp1()) || op1->isContainedCompareChainSegment(op1->gtGetOp2()))) - { - return cmp; - } -#endif - #ifdef TARGET_XARCH var_types op1Type = op1->TypeGet(); if (IsContainableMemoryOp(op1) && varTypeIsSmall(op1Type) && FitsIn(op1Type, op2Value)) @@ -3408,7 +3469,6 @@ GenTree* Lowering::LowerSelect(GenTreeConditional* select) } } -#ifdef TARGET_XARCH // Do not transform GT_SELECT with GTF_SET_FLAGS into GT_SELECTCC; this // node is used by decomposition on x86. // TODO-CQ: If we allowed multiple nodes to consume the same CPU flags then @@ -3423,7 +3483,6 @@ GenTree* Lowering::LowerSelect(GenTreeConditional* select) ContainCheckSelect(newSelect); return newSelect->gtNext; } -#endif ContainCheckSelect(select); return select->gtNext; @@ -3519,21 +3578,30 @@ bool Lowering::TryLowerConditionToFlagsNode(GenTree* parent, GenTree* condition, return true; } - // TODO-Cleanup: Avoid creating these SETCC nodes in the first place. if (condition->OperIs(GT_SETCC)) { assert((condition->gtPrev->gtFlags & GTF_SET_FLAGS) != 0); - GenTree* flagsProducer = condition->gtPrev; - if (!IsInvariantInRange(flagsProducer, parent, condition)) + GenTree* flagsDef = condition->gtPrev; +#ifdef TARGET_ARM64 + // CCMP is a flag producing node that also consumes flags, so find the + // "root" of the flags producers and move the entire range. + // We limit this to 10 nodes look back to avoid quadratic behavior. + for (int i = 0; i < 10 && flagsDef->OperIs(GT_CCMP); i++) + { + assert((flagsDef->gtPrev != nullptr) && ((flagsDef->gtPrev->gtFlags & GTF_SET_FLAGS) != 0)); + flagsDef = flagsDef->gtPrev; + } +#endif + if (!IsRangeInvariantInRange(flagsDef, condition->gtPrev, parent, condition)) { return false; } *cond = condition->AsCC()->gtCondition; + LIR::Range range = BlockRange().Remove(flagsDef, condition->gtPrev); + BlockRange().InsertBefore(parent, std::move(range)); BlockRange().Remove(condition); - BlockRange().Remove(flagsProducer); - BlockRange().InsertBefore(parent, flagsProducer); return true; } diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index ae3da6f7155ef..5ff37fd9e9c6b 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -85,10 +85,9 @@ class Lowering final : public Phase void ContainCheckLclHeap(GenTreeOp* node); void ContainCheckRet(GenTreeUnOp* ret); #ifdef TARGET_ARM64 - bool IsValidCompareChain(GenTree* child, GenTree* parent); - bool ContainCheckCompareChain(GenTree* child, GenTree* parent, GenTree** earliestValid); - void ContainCheckCompareChainForAnd(GenTree* tree); - void ContainCheckConditionalCompare(GenTreeOp* cmp); + GenTree* TryLowerAndToCCMP(GenTreeOp* tree); + insCflags TruthifyingFlags(GenCondition cond); + void ContainCheckConditionalCompare(GenTreeCCMP* ccmp); void ContainCheckNeg(GenTreeOp* neg); #endif void ContainCheckSelect(GenTreeOp* select); @@ -504,6 +503,10 @@ class Lowering final : public Phase bool IsInvariantInRange(GenTree* node, GenTree* endExclusive) const; bool IsInvariantInRange(GenTree* node, GenTree* endExclusive, GenTree* ignoreNode) const; + bool IsRangeInvariantInRange(GenTree* rangeStart, + GenTree* rangeEnd, + GenTree* endExclusive, + GenTree* ignoreNode) const; // Checks for memory conflicts in the instructions between childNode and parentNode, and returns true if childNode // can be contained. diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index c7ce2e4741bef..80c459ffc5e98 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -482,7 +482,11 @@ GenTree* Lowering::LowerBinaryArithmetic(GenTreeOp* binOp) #ifdef TARGET_ARM64 else { - ContainCheckCompareChainForAnd(binOp); + GenTree* next = TryLowerAndToCCMP(binOp); + if (next != nullptr) + { + return next; + } } #endif } @@ -2264,156 +2268,116 @@ void Lowering::ContainCheckCompare(GenTreeOp* cmp) #ifdef TARGET_ARM64 //------------------------------------------------------------------------ -// IsValidCompareChain : Determine if the node contains a valid chain of ANDs and CMPs. +// TryLowerAndToCCMP : Lower an and of two conditions into test + CCMP + SETCC nodes. // // Arguments: -// child - pointer to the node being checked. -// parent - parent node of the child. -// -// Return value: -// True if a valid chain is found. +// tree - pointer to the node // -// Notes: -// A compare chain is a sequence of CMP nodes connected by AND nodes. -// For example: AND (AND (CMP A B) (CMP C D)) (CMP E F) -// The chain can just be a single compare node, however it's parent -// must always be an AND or SELECT node. -// If a CMP or AND node is contained then it and all it's children are -// considered to be in a valid chain. -// Chains are built up during the lowering of each successive parent. -// -bool Lowering::IsValidCompareChain(GenTree* child, GenTree* parent) +GenTree* Lowering::TryLowerAndToCCMP(GenTreeOp* tree) { - assert(parent->OperIs(GT_AND) || parent->OperIs(GT_SELECT)); + assert(tree->OperIs(GT_AND)); - if (parent->isContainedCompareChainSegment(child)) + if (!comp->opts.OptimizationEnabled()) { - // Already have a chain. - return true; + return nullptr; } - else if (child->OperIs(GT_AND)) + + GenTree* op1 = tree->gtGetOp1(); + GenTree* op2 = tree->gtGetOp2(); + + // Find out whether op2 is eligible to be converted to a conditional + // compare. It must be a normal integral relop; for example, we cannot + // conditionally perform a floating point comparison and there is no "ctst" + // instruction that would allow us to conditionally implement + // TEST_EQ/TEST_NE. + // + if (!op2->OperIsCmpCompare() || !varTypeIsIntegral(op2->gtGetOp1())) { - // Count both sides. - return IsValidCompareChain(child->AsOp()->gtGetOp2(), child) && - IsValidCompareChain(child->AsOp()->gtGetOp1(), child); + return nullptr; } - else if (child->OperIsCmpCompare() && varTypeIsIntegral(child->gtGetOp1()) && varTypeIsIntegral(child->gtGetOp2())) + + // For op1 we can allow more arbitrary operations that set the condition + // flags; the final transformation into the flags def is done by + // TryLowerConditionToFlagsNode below, but we have a quick early out here + // too. + // + if (!op1->OperIsCompare() && !op1->OperIs(GT_SETCC)) { - // Can the child compare be contained. - return IsInvariantInRange(child, parent); + return nullptr; } - return false; -} - -//------------------------------------------------------------------------ -// ContainCheckCompareChain : Determine if a chain of ANDs and CMPs can be contained. -// -// Arguments: -// child - pointer to the node being checked. -// parent - parent node of the child. -// startOfChain - If found, returns the earliest valid op in the chain. -// -// Return value: -// True if a valid chain is was contained. -// -// Notes: -// Assumes the chain was checked via IsValidCompareChain. -// -bool Lowering::ContainCheckCompareChain(GenTree* child, GenTree* parent, GenTree** startOfChain) -{ - assert(parent->OperIs(GT_AND) || parent->OperIs(GT_SELECT)); - *startOfChain = nullptr; // Nothing found yet. - - if (parent->isContainedCompareChainSegment(child)) + // We leave checking invariance of op1 to tree to TryLowerConditionToFlagsNode. + if (!IsInvariantInRange(op2, tree)) { - // Already have a contained chain. - return true; + return nullptr; } - // Can the child be contained. - else if (IsInvariantInRange(child, parent)) + + GenCondition cond1; + if (!TryLowerConditionToFlagsNode(tree, op1, &cond1)) { - if (child->OperIs(GT_AND)) - { - // If Op2 is not contained, then try to contain it. - if (!child->isContainedCompareChainSegment(child->AsOp()->gtGetOp2())) - { - if (!ContainCheckCompareChain(child->gtGetOp2(), child, startOfChain)) - { - // Op2 must be contained in order to contain Op1 or the AND. - return false; - } - } + return nullptr; + } - // If Op1 is not contained, then try to contain it. - if (!child->isContainedCompareChainSegment(child->AsOp()->gtGetOp1())) - { - if (!ContainCheckCompareChain(child->gtGetOp1(), child, startOfChain)) - { - return false; - } - } + BlockRange().Remove(op2); + BlockRange().InsertBefore(tree, op2); - // Contain the AND. - child->SetContained(); - return true; - } - else if (child->OperIsCmpCompare()) - { - child->AsOp()->SetContained(); + GenCondition cond2 = GenCondition::FromRelop(op2); + op2->SetOper(GT_CCMP); + op2->gtType = TYP_VOID; + op2->gtFlags |= GTF_SET_FLAGS; - // Ensure the children of the compare are contained correctly. - child->AsOp()->gtGetOp1()->ClearContained(); - child->AsOp()->gtGetOp2()->ClearContained(); - ContainCheckConditionalCompare(child->AsOp()); - *startOfChain = child; - return true; - } - } + op2->gtGetOp1()->ClearContained(); + op2->gtGetOp2()->ClearContained(); - return false; + GenTreeCCMP* ccmp = op2->AsCCMP(); + ccmp->gtCondition = cond1; + // If the first comparison fails, set the condition flags to something that + // makes the second one fail as well so that the overall AND failed. + ccmp->gtFlagsVal = TruthifyingFlags(GenCondition::Reverse(cond2)); + ContainCheckConditionalCompare(ccmp); + + tree->SetOper(GT_SETCC); + tree->AsCC()->gtCondition = cond2; + + return tree->gtNext; } //------------------------------------------------------------------------ -// ContainCheckCompareChainForAnd : Determine if an AND is a containable chain +// TruthifyingFlags: Get a flags immediate that will make a specified condition true. // // Arguments: -// node - pointer to the node +// condition - the condition. +// +// Returns: +// A flags immediate that, if those flags were set, would cause the specified condition to be true. // -void Lowering::ContainCheckCompareChainForAnd(GenTree* tree) +insCflags Lowering::TruthifyingFlags(GenCondition condition) { - assert(tree->OperIs(GT_AND)); - - if (!comp->opts.OptimizationEnabled()) - { - return; - } - - // First check there is a valid chain. - if (IsValidCompareChain(tree->AsOp()->gtGetOp2(), tree) && IsValidCompareChain(tree->AsOp()->gtGetOp1(), tree)) - { - GenTree* startOfChain = nullptr; - - // To ensure ordering at code generation, Op1 and the parent can - // only be contained if Op2 is contained. - if (ContainCheckCompareChain(tree->AsOp()->gtGetOp2(), tree, &startOfChain)) - { - if (ContainCheckCompareChain(tree->AsOp()->gtGetOp1(), tree, &startOfChain)) - { - // If op1 is the start of a chain, then it'll be generated as a standard compare. - if (startOfChain != nullptr) - { - // The earliest node in the chain will be generated as a standard compare. - assert(startOfChain->OperIsCmpCompare()); - startOfChain->AsOp()->gtGetOp1()->ClearContained(); - startOfChain->AsOp()->gtGetOp2()->ClearContained(); - ContainCheckCompare(startOfChain->AsOp()); - } - } - } - - JITDUMP("Lowered `AND` chain:\n"); - DISPTREE(tree); + switch (condition.GetCode()) + { + case GenCondition::EQ: + return INS_FLAGS_Z; + case GenCondition::NE: + return INS_FLAGS_NONE; + case GenCondition::SGE: + return INS_FLAGS_Z; + case GenCondition::SGT: + return INS_FLAGS_NONE; + case GenCondition::SLT: + return INS_FLAGS_NC; + case GenCondition::SLE: + return INS_FLAGS_NZC; + case GenCondition::UGE: + return INS_FLAGS_C; + case GenCondition::UGT: + return INS_FLAGS_C; + case GenCondition::ULT: + return INS_FLAGS_NONE; + case GenCondition::ULE: + return INS_FLAGS_Z; + default: + NO_WAY("unexpected condition type"); + return INS_FLAGS_NONE; } } @@ -2423,9 +2387,8 @@ void Lowering::ContainCheckCompareChainForAnd(GenTree* tree) // Arguments: // node - pointer to the node // -void Lowering::ContainCheckConditionalCompare(GenTreeOp* cmp) +void Lowering::ContainCheckConditionalCompare(GenTreeCCMP* cmp) { - assert(cmp->OperIsCmpCompare()); GenTree* op2 = cmp->gtOp2; if (op2->IsCnsIntOrI() && !op2->AsIntCon()->ImmedValNeedsReloc(comp)) @@ -2452,38 +2415,8 @@ void Lowering::ContainCheckSelect(GenTreeOp* node) #ifdef TARGET_ARM noway_assert(!"GT_SELECT nodes are not supported on arm32"); #else - if (!comp->opts.OptimizationEnabled()) - { - return; - } - - GenTree* cond = node->AsConditional()->gtCond; - GenTree* op1 = node->gtOp1; - GenTree* op2 = node->gtOp2; - - if (cond->OperIsCompare()) - { - // All compare node types (including TEST_) are containable. - if (IsInvariantInRange(cond, node)) - { - cond->AsOp()->SetContained(); - } - } - else - { - // Check for a compare chain and try to contain it. - GenTree* startOfChain = nullptr; - ContainCheckCompareChain(cond, node, &startOfChain); - - if (startOfChain != nullptr) - { - // The earliest node in the chain will be generated as a standard compare. - assert(startOfChain->OperIsCmpCompare()); - startOfChain->AsOp()->gtGetOp1()->ClearContained(); - startOfChain->AsOp()->gtGetOp2()->ClearContained(); - ContainCheckCompare(startOfChain->AsOp()); - } - } + GenTree* op1 = node->gtOp1; + GenTree* op2 = node->gtOp2; if (op1->IsIntegralConst(0)) { diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 5226e28ad33e6..7c2cbee140280 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -403,6 +403,7 @@ int LinearScan::BuildNode(GenTree* tree) case GT_TEST_NE: case GT_CMP: case GT_TEST: + case GT_CCMP: case GT_JCMP: srcCount = BuildCmp(tree); break; @@ -772,6 +773,10 @@ int LinearScan::BuildNode(GenTree* tree) assert(dstCount == 1); srcCount = BuildSelect(tree->AsConditional()); break; + case GT_SELECTCC: + assert(dstCount == 1); + srcCount = BuildSelect(tree->AsOp()); + break; } // end switch (tree->OperGet()) diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index 084861949d703..bc3cf04c23594 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -833,9 +833,14 @@ int LinearScan::BuildCast(GenTreeCast* cast) // int LinearScan::BuildSelect(GenTreeOp* select) { - assert(select->OperIs(GT_SELECT)); + assert(select->OperIs(GT_SELECT, GT_SELECTCC)); + + int srcCount = 0; + if (select->OperIs(GT_SELECT)) + { + srcCount += BuildOperandUses(select->AsConditional()->gtCond); + } - int srcCount = BuildOperandUses(select->AsConditional()->gtCond); srcCount += BuildOperandUses(select->gtOp1); srcCount += BuildOperandUses(select->gtOp2); BuildDef(select); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 70dc5dc1648b2..14f142d7908fa 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -4108,8 +4108,10 @@ int LinearScan::BuildGCWriteBarrier(GenTree* tree) // int LinearScan::BuildCmp(GenTree* tree) { -#ifdef TARGET_XARCH +#if defined(TARGET_XARCH) assert(tree->OperIsCompare() || tree->OperIs(GT_CMP, GT_TEST, GT_JCMP, GT_BT)); +#elif defined(TARGET_ARM64) + assert(tree->OperIsCompare() || tree->OperIs(GT_CMP, GT_TEST, GT_JCMP, GT_CCMP)); #else assert(tree->OperIsCompare() || tree->OperIs(GT_CMP, GT_TEST, GT_JCMP)); #endif