diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index ead90a27dcd29..7ad20ff1c228b 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -752,7 +752,7 @@ void Compiler::fgMorphCallInline(GenTreeCall* call, InlineResult* inlineResult) { bool inliningFailed = false; - InlineCandidateInfo* inlCandInfo = call->gtInlineCandidateInfo; + InlineCandidateInfo* inlCandInfo = call->GetInlineCandidateInfo(); // Is this call an inline candidate? if (call->IsInlineCandidate()) @@ -778,7 +778,7 @@ void Compiler::fgMorphCallInline(GenTreeCall* call, InlineResult* inlineResult) #ifdef DEBUG // In debug we always put all inline attempts into the inline tree. InlineContext* ctx = - m_inlineStrategy->NewContext(call->gtInlineCandidateInfo->inlinersContext, fgMorphStmt, call); + m_inlineStrategy->NewContext(call->GetInlineCandidateInfo()->inlinersContext, fgMorphStmt, call); ctx->SetFailed(inlineResult); #endif } @@ -1045,7 +1045,7 @@ void Compiler::fgInvokeInlineeCompiler(GenTreeCall* call, InlineResult* inlineRe inlineInfo.hasSIMDTypeArgLocalOrReturn = false; #endif // FEATURE_SIMD - InlineCandidateInfo* inlineCandidateInfo = call->gtInlineCandidateInfo; + InlineCandidateInfo* inlineCandidateInfo = call->GetInlineCandidateInfo(); noway_assert(inlineCandidateInfo); // Store the link to inlineCandidateInfo into inlineInfo inlineInfo.inlineCandidateInfo = inlineCandidateInfo; diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 684d374ceb4d1..3529a344ea0fe 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -2179,6 +2179,64 @@ GenTree* Compiler::getArrayLengthFromAllocation(GenTree* tree DEBUGARG(BasicBloc return arrayLength; } +//------------------------------------------------------------------------- +// SetSingleInlineCadidateInfo: set a single inline candidate info in the current call. +// +// Arguments: +// candidateInfo - inline candidate info +// +void GenTreeCall::SetSingleInlineCadidateInfo(InlineCandidateInfo* candidateInfo) +{ + if (candidateInfo != nullptr) + { + gtFlags |= GTF_CALL_INLINE_CANDIDATE; + gtInlineInfoCount = 1; + } + else + { + gtInlineInfoCount = 0; + gtFlags &= ~GTF_CALL_INLINE_CANDIDATE; + gtCallMoreFlags &= ~GTF_CALL_M_GUARDED_DEVIRT; + } + gtInlineCandidateInfo = candidateInfo; +} + +//------------------------------------------------------------------------- +// GetGDVCandidateInfo: Get GDV candidate info in the current call by index. +// +// Return Value: +// GDV candidate info +// +InlineCandidateInfo* GenTreeCall::GetGDVCandidateInfo(uint8_t index) +{ + assert(index < gtInlineInfoCount); + return >InlineCandidateInfo[index]; +} + +//------------------------------------------------------------------------- +// AddGDVCandidateInfo: Record a guarded devirtualization (GDV) candidate info +// for this call. For now, we only support one GDV candidate per call. +// +// Arguments: +// candidateInfo - GDV candidate info +// +void GenTreeCall::AddGDVCandidateInfo(InlineCandidateInfo* candidateInfo) +{ + assert(candidateInfo != nullptr); + if (gtInlineInfoCount == 0) + { + gtInlineCandidateInfo = candidateInfo; + } + else + { + // Allocate a fixed list of InlineCandidateInfo structs + assert(!"multiple GDV candidates are not implemented yet"); + } + + gtCallMoreFlags |= GTF_CALL_M_GUARDED_DEVIRT; + gtInlineInfoCount++; +} + //------------------------------------------------------------------------- // HasSideEffects: // Returns true if this call has any side effects. All non-helpers are considered to have side-effects. Only helpers @@ -7758,7 +7816,7 @@ GenTreeCall* Compiler::gtNewCallNode(gtCallTypes callType, } else { - node->gtInlineCandidateInfo = nullptr; + node->ClearInlineInfo(); } node->gtReturnType = type; @@ -9409,6 +9467,7 @@ GenTreeCall* Compiler::gtCloneExprCallHelper(GenTreeCall* tree, { copy->gtCallMethHnd = tree->gtCallMethHnd; copy->gtInlineCandidateInfo = tree->gtInlineCandidateInfo; + copy->gtInlineInfoCount = tree->gtInlineInfoCount; } copy->gtCallType = tree->gtCallType; @@ -12419,10 +12478,10 @@ void Compiler::gtDispTree(GenTree* tree, printf(" (FramesRoot last use)"); } - if (((call->gtFlags & GTF_CALL_INLINE_CANDIDATE) != 0) && (call->gtInlineCandidateInfo != nullptr) && - (call->gtInlineCandidateInfo->exactContextHnd != nullptr)) + if (((call->gtFlags & GTF_CALL_INLINE_CANDIDATE) != 0) && (call->GetInlineCandidateInfo() != nullptr) && + (call->GetInlineCandidateInfo()->exactContextHnd != nullptr)) { - printf(" (exactContextHnd=0x%p)", dspPtr(call->gtInlineCandidateInfo->exactContextHnd)); + printf(" (exactContextHnd=0x%p)", dspPtr(call->GetInlineCandidateInfo()->exactContextHnd)); } gtDispCommonEndLine(tree); @@ -17991,7 +18050,7 @@ CORINFO_CLASS_HANDLE Compiler::gtGetClassHandle(GenTree* tree, bool* pIsExact, b // type class handle in the inline info (for GDV candidates, // this data is valid only for a correct guess, so we cannot // use it). - InlineCandidateInfo* inlInfo = call->gtInlineCandidateInfo; + InlineCandidateInfo* inlInfo = call->GetInlineCandidateInfo(); assert(inlInfo != nullptr); // Grab it as our first cut at a return type. diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index a050f2d6e2bb4..3462654705207 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -172,7 +172,6 @@ enum TargetHandleType : BYTE struct BasicBlock; enum BasicBlockFlags : unsigned __int64; struct InlineCandidateInfo; -struct GuardedDevirtualizationCandidateInfo; struct HandleHistogramProfileCandidateInfo; struct LateDevirtualizationInfo; @@ -5375,11 +5374,6 @@ struct GenTreeCall final : public GenTree gtCallMoreFlags &= ~GTF_CALL_M_GUARDED_DEVIRT; } - void SetGuardedDevirtualizationCandidate() - { - gtCallMoreFlags |= GTF_CALL_M_GUARDED_DEVIRT; - } - void SetIsGuarded() { gtCallMoreFlags |= GTF_CALL_M_GUARDED; @@ -5435,6 +5429,22 @@ struct GenTreeCall final : public GenTree return (gtCallMoreFlags & GTF_CALL_M_RETBUFFARG_LCLOPT) != 0; } + InlineCandidateInfo* GetInlineCandidateInfo() + { + return gtInlineCandidateInfo; + } + + void SetSingleInlineCadidateInfo(InlineCandidateInfo* candidateInfo); + + InlineCandidateInfo* GetGDVCandidateInfo(uint8_t index = 0); + + void AddGDVCandidateInfo(InlineCandidateInfo* candidateInfo); + + void ClearInlineInfo() + { + SetSingleInlineCadidateInfo(nullptr); + } + //----------------------------------------------------------------------------------------- // GetIndirectionCellArgKind: Get the kind of indirection cell used by this call. // @@ -5505,10 +5515,13 @@ struct GenTreeCall final : public GenTree return mayUseDispatcher && shouldUseDispatcher ? CFGCallKind::Dispatch : CFGCallKind::ValidateAndCall; } - GenTreeCallFlags gtCallMoreFlags; // in addition to gtFlags - gtCallTypes gtCallType : 3; // value from the gtCallTypes enumeration - var_types gtReturnType : 5; // exact return type - CORINFO_CLASS_HANDLE gtRetClsHnd; // The return type handle of the call if it is a struct; always available + GenTreeCallFlags gtCallMoreFlags; // in addition to gtFlags + gtCallTypes gtCallType : 3; // value from the gtCallTypes enumeration + var_types gtReturnType : 5; // exact return type + + uint8_t gtInlineInfoCount; // number of inline candidates for the given call + + CORINFO_CLASS_HANDLE gtRetClsHnd; // The return type handle of the call if it is a struct; always available union { void* gtStubCallStubAddr; // GTF_CALL_VIRT_STUB - these are never inlined CORINFO_CLASS_HANDLE gtInitClsHnd; // Used by static init helpers, represents a class they init @@ -5518,10 +5531,9 @@ struct GenTreeCall final : public GenTree // only used for CALLI unmanaged calls (CT_INDIRECT) GenTree* gtCallCookie; // gtInlineCandidateInfo is only used when inlining methods - InlineCandidateInfo* gtInlineCandidateInfo; - GuardedDevirtualizationCandidateInfo* gtGuardedDevirtualizationCandidateInfo; - HandleHistogramProfileCandidateInfo* gtHandleHistogramProfileCandidateInfo; - LateDevirtualizationInfo* gtLateDevirtualizationInfo; + InlineCandidateInfo* gtInlineCandidateInfo; + HandleHistogramProfileCandidateInfo* gtHandleHistogramProfileCandidateInfo; + LateDevirtualizationInfo* gtLateDevirtualizationInfo; CORINFO_GENERIC_HANDLE compileTimeHelperArgumentHandle; // Used to track type handle argument of dynamic helpers void* gtDirectCallAddress; // Used to pass direct call address between lower and codegen }; diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 6002a39dc7bb1..5d966c6122dbd 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -1702,7 +1702,7 @@ bool Compiler::impSpillStackEntry(unsigned level, { JITDUMP("\n*** see V%02u = GT_RET_EXPR, noting temp\n", tnum); GenTree* call = tree->AsRetExpr()->gtInlineCandidate; - InlineCandidateInfo* ici = call->AsCall()->gtInlineCandidateInfo; + InlineCandidateInfo* ici = call->AsCall()->GetInlineCandidateInfo(); ici->preexistingSpillTemp = tnum; } } diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index a676724e4bd5d..61af557d583c2 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -1351,7 +1351,7 @@ var_types Compiler::impImportCall(OPCODE opcode, GenTreeRetExpr* retExpr = gtNewInlineCandidateReturnExpr(call->AsCall(), genActualType(callRetTyp)); // Link the retExpr to the call so if necessary we can manipulate it later. - origCall->gtInlineCandidateInfo->retExpr = retExpr; + origCall->GetInlineCandidateInfo()->retExpr = retExpr; // Propagate retExpr as the placeholder for the call. call = retExpr; @@ -6059,7 +6059,6 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, classHandle != NO_CLASS_HANDLE ? "class" : "method", classHandle != NO_CLASS_HANDLE ? eeGetClassName(classHandle) : eeGetMethodFullName(methodHandle)); setMethodHasGuardedDevirtualization(); - call->SetGuardedDevirtualizationCandidate(); // Spill off any GT_RET_EXPR subtrees so we can clone the call. // @@ -6069,7 +6068,7 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, // Gather some information for later. Note we actually allocate InlineCandidateInfo // here, as the devirtualized half of this call will likely become an inline candidate. // - GuardedDevirtualizationCandidateInfo* pInfo = new (this, CMK_Inlining) InlineCandidateInfo; + InlineCandidateInfo* pInfo = new (this, CMK_Inlining) InlineCandidateInfo; pInfo->guardedMethodHandle = methodHandle; pInfo->guardedMethodUnboxedEntryHandle = nullptr; @@ -6094,7 +6093,7 @@ void Compiler::addGuardedDevirtualizationCandidate(GenTreeCall* call, } } - call->gtGuardedDevirtualizationCandidateInfo = pInfo; + call->AddGDVCandidateInfo(pInfo); } //------------------------------------------------------------------------ @@ -6270,13 +6269,13 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, if (call->IsGuardedDevirtualizationCandidate()) { - if (call->gtGuardedDevirtualizationCandidateInfo->guardedMethodUnboxedEntryHandle != nullptr) + if (call->GetGDVCandidateInfo()->guardedMethodUnboxedEntryHandle != nullptr) { - fncHandle = call->gtGuardedDevirtualizationCandidateInfo->guardedMethodUnboxedEntryHandle; + fncHandle = call->GetGDVCandidateInfo()->guardedMethodUnboxedEntryHandle; } else { - fncHandle = call->gtGuardedDevirtualizationCandidateInfo->guardedMethodHandle; + fncHandle = call->GetGDVCandidateInfo()->guardedMethodHandle; } methAttr = info.compCompHnd->getMethodAttribs(fncHandle); } @@ -6377,13 +6376,13 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, } // The old value should be null OR this call should be a guarded devirtualization candidate. - assert((call->gtInlineCandidateInfo == nullptr) || call->IsGuardedDevirtualizationCandidate()); + assert((call->GetInlineCandidateInfo() == nullptr) || call->IsGuardedDevirtualizationCandidate()); // The new value should not be null. assert(inlineCandidateInfo != nullptr); inlineCandidateInfo->exactContextNeedsRuntimeLookup = exactContextNeedsRuntimeLookup; inlineCandidateInfo->ilOffset = ilOffset; - call->gtInlineCandidateInfo = inlineCandidateInfo; + call->SetSingleInlineCadidateInfo(inlineCandidateInfo); // If we're in an inlinee compiler, and have a return spill temp, and this inline candidate // is also a tail call candidate, it can use the same return spill temp. @@ -6396,9 +6395,6 @@ void Compiler::impMarkInlineCandidateHelper(GenTreeCall* call, inlineCandidateInfo->preexistingSpillTemp); } - // Mark the call node as inline candidate. - call->gtFlags |= GTF_CALL_INLINE_CANDIDATE; - // Let the strategy know there's another candidate. impInlineRoot()->m_inlineStrategy->NoteCandidate(); @@ -6895,7 +6891,7 @@ void Compiler::impDevirtualizeCall(GenTreeCall* call, // Clear the inline candidate info (may be non-null since // it's a union field used for other things by virtual // stubs) - call->gtInlineCandidateInfo = nullptr; + call->ClearInlineInfo(); call->gtCallMoreFlags &= ~GTF_CALL_M_HAS_LATE_DEVIRT_INFO; #if defined(DEBUG) @@ -7675,7 +7671,7 @@ void Compiler::impCheckCanInline(GenTreeCall* call, if (pParam->call->IsGuardedDevirtualizationCandidate()) { - pInfo = pParam->call->gtInlineCandidateInfo; + pInfo = pParam->call->GetInlineCandidateInfo(); } else { diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index b5183f1586e59..fae20cb92c7cc 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -465,7 +465,7 @@ class IndirectCallTransformer return; } - likelihood = origCall->gtGuardedDevirtualizationCandidateInfo->likelihood; + likelihood = origCall->GetGDVCandidateInfo()->likelihood; assert((likelihood >= 0) && (likelihood <= 100)); JITDUMP("Likelihood of correct guess is %u\n", likelihood); @@ -574,7 +574,7 @@ class IndirectCallTransformer // lastStmt = checkBlock->lastStmt(); - GuardedDevirtualizationCandidateInfo* guardedInfo = origCall->gtGuardedDevirtualizationCandidateInfo; + InlineCandidateInfo* guardedInfo = origCall->GetGDVCandidateInfo(); // Create comparison. On success we will jump to do the indirect call. GenTree* compare; @@ -655,7 +655,7 @@ class IndirectCallTransformer // // Note implicit by-ref returns should have already been converted // so any struct copy we induce here should be cheap. - InlineCandidateInfo* const inlineInfo = origCall->gtInlineCandidateInfo; + InlineCandidateInfo* const inlineInfo = origCall->GetInlineCandidateInfo(); if (!origCall->TypeIs(TYP_VOID)) { @@ -736,7 +736,7 @@ class IndirectCallTransformer { thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock); thenBlock->bbFlags |= currBlock->bbFlags & BBF_SPLIT_GAINED; - InlineCandidateInfo* inlineInfo = origCall->gtInlineCandidateInfo; + InlineCandidateInfo* inlineInfo = origCall->GetInlineCandidateInfo(); CORINFO_CLASS_HANDLE clsHnd = inlineInfo->guardedClassHandle; // @@ -840,7 +840,7 @@ class IndirectCallTransformer "inlineable\n"); call->gtFlags &= ~GTF_CALL_INLINE_CANDIDATE; - call->gtInlineCandidateInfo = nullptr; + call->ClearInlineInfo(); if (returnTemp != BAD_VAR_NUM) { @@ -864,7 +864,7 @@ class IndirectCallTransformer inlineInfo->clsHandle = compiler->info.compCompHnd->getMethodClass(methodHnd); inlineInfo->exactContextHnd = context; inlineInfo->preexistingSpillTemp = returnTemp; - call->gtInlineCandidateInfo = inlineInfo; + call->SetSingleInlineCadidateInfo(inlineInfo); // If there was a ret expr for this call, we need to create a new one // and append it just after the call. @@ -1092,11 +1092,11 @@ class IndirectCallTransformer GenTreeCall* const call = root->AsCall(); if (call->IsGuardedDevirtualizationCandidate() && - (call->gtGuardedDevirtualizationCandidateInfo->likelihood >= gdvChainLikelihood)) + (call->GetGDVCandidateInfo()->likelihood >= gdvChainLikelihood)) { JITDUMP("GDV call at [%06u] has likelihood %u >= %u; chaining (%u stmts, %u nodes to dup).\n", - compiler->dspTreeID(call), call->gtGuardedDevirtualizationCandidateInfo->likelihood, - gdvChainLikelihood, chainStatementDup, chainNodeDup); + compiler->dspTreeID(call), call->GetGDVCandidateInfo()->likelihood, gdvChainLikelihood, + chainStatementDup, chainNodeDup); call->gtCallMoreFlags |= GTF_CALL_M_GUARDED_DEVIRT_CHAIN; break; diff --git a/src/coreclr/jit/inline.cpp b/src/coreclr/jit/inline.cpp index 9c28d575e446d..a9fffc513fe2a 100644 --- a/src/coreclr/jit/inline.cpp +++ b/src/coreclr/jit/inline.cpp @@ -1281,7 +1281,7 @@ InlineContext* InlineStrategy::NewContext(InlineContext* parentContext, Statemen if (call->IsInlineCandidate()) { - InlineCandidateInfo* info = call->gtInlineCandidateInfo; + InlineCandidateInfo* info = call->GetInlineCandidateInfo(); context->m_Code = info->methInfo.ILCode; context->m_ILSize = info->methInfo.ILCodeSize; context->m_ActualCallOffset = info->ilOffset; diff --git a/src/coreclr/jit/inline.h b/src/coreclr/jit/inline.h index 74c337e998f24..e350393cfd29c 100644 --- a/src/coreclr/jit/inline.h +++ b/src/coreclr/jit/inline.h @@ -586,26 +586,19 @@ struct HandleHistogramProfileCandidateInfo unsigned probeIndex; }; -// GuardedDevirtualizationCandidateInfo provides information about -// a potential target of a virtual or interface call. +// InlineCandidateInfo provides basic information about a particular +// inline candidate. +// +// Calls can start out as GDV candidates and turn into inline candidates // -struct GuardedDevirtualizationCandidateInfo : HandleHistogramProfileCandidateInfo +struct InlineCandidateInfo : public HandleHistogramProfileCandidateInfo { CORINFO_CLASS_HANDLE guardedClassHandle; CORINFO_METHOD_HANDLE guardedMethodHandle; CORINFO_METHOD_HANDLE guardedMethodUnboxedEntryHandle; unsigned likelihood; bool requiresInstMethodTableArg; -}; -// InlineCandidateInfo provides basic information about a particular -// inline candidate. -// -// It is a superset of GuardedDevirtualizationCandidateInfo: calls -// can start out as GDv candidates and turn into inline candidates -// -struct InlineCandidateInfo : public GuardedDevirtualizationCandidateInfo -{ CORINFO_METHOD_INFO methInfo; // the logical IL caller of this inlinee. diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 71469e791b48b..4759b2b30db49 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -187,13 +187,13 @@ GenTree* Compiler::fgMorphIntoHelperCall(GenTree* tree, int helper, bool morphAr GenTreeCall* call = tree->AsCall(); // Args are cleared by ChangeOper above - call->gtCallType = CT_HELPER; - call->gtReturnType = tree->TypeGet(); - call->gtCallMethHnd = eeFindHelper(helper); - call->gtRetClsHnd = nullptr; - call->gtCallMoreFlags = GTF_CALL_M_EMPTY; - call->gtInlineCandidateInfo = nullptr; - call->gtControlExpr = nullptr; + call->gtCallType = CT_HELPER; + call->gtReturnType = tree->TypeGet(); + call->gtCallMethHnd = eeFindHelper(helper); + call->gtRetClsHnd = nullptr; + call->gtCallMoreFlags = GTF_CALL_M_EMPTY; + call->gtControlExpr = nullptr; + call->ClearInlineInfo(); #ifdef UNIX_X86_ABI call->gtFlags |= GTF_CALL_POP_ARGS; #endif // UNIX_X86_ABI