Initial version of class profiling for PGO (#45133)

* Initial version of class profiling for PGO

Add support to the jit and runtime so that PGO can determine the distribution of
classes at virtual and indirect call sites.

Use this information when jitting to enable guarded devirtualization, if there
is a suitably likely class to guess for.

Enable by setting:
```
COMPlus_TieredCompilation=1
COMPlus_TieredPGO=1
COMPlus_JitClassProfiling=1
COMPlus_JitEnableGuardedDevirtualization=1
```
impact can be enhanced by also setting
```
COMPlus_TC_QuickJitForLoops=1
```
to allow more methods to pass through Tier0.

eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -472,6 +472,7 @@ jobs:
           - jitobjectstackallocation
           - jitpgo
           - jitpgo_inline
+          - jitpgo_classes
         ${{ if in(parameters.testGroup, 'ilasm') }}:
           scenarios:
           - ilasmroundtrip

src/coreclr/src/ToolBox/superpmi/superpmi-shared/icorjitinfoimpl.h

@@ -979,6 +979,34 @@ struct BlockCounts  // Also defined here: code:CORBBTPROF_BLOCK_DATA
     UINT32 ILOffset;
     UINT32 ExecutionCount;
 };
+
+// Data structure for a single class probe.
+//
+// ILOffset is the IL offset in the method for the call site being probed.
+// Currently it must be ORed with CLASS_FLAG and (for interface calls)
+// INTERFACE_FLAG.
+//
+// Count is the number of times a call was made at that call site.
+//
+// SIZE is the number of entries in the table.
+//
+// SAMPLE_INTERVAL must be >= SIZE. SAMPLE_INTERVAL / SIZE
+// gives the average number of calls between table updates.
+// 
+struct ClassProfile
+{
+    enum { 
+        SIZE = 8, 
+        SAMPLE_INTERVAL = 32, 
+        CLASS_FLAG     = 0x80000000, 
+        INTERFACE_FLAG = 0x40000000,
+        OFFSET_MASK    = 0x3FFFFFFF
+    };
+
+    UINT32 ILOffset;
+    UINT32 Count;
+    CORINFO_CLASS_HANDLE ClassTable[SIZE];
+};
 */
 
 // allocate a basic block profile buffer where execution counts will be stored
@@ -993,6 +1021,15 @@ HRESULT getMethodBlockCounts(CORINFO_METHOD_HANDLE ftnHnd,
                              BlockCounts**     pBlockCounts,
                              UINT32 *          pNumRuns);
 
+// Get the likely implementing class for a virtual call or interface call made by ftnHnd
+// at the indicated IL offset. baseHnd is the interface class or base class for the method
+// being called. 
+CORINFO_CLASS_HANDLE getLikelyClass(CORINFO_METHOD_HANDLE ftnHnd,
+                                    CORINFO_CLASS_HANDLE  baseHnd,
+                                    UINT32                ilOffset,
+                                    UINT32*               pLikelihood,
+                                    UINT32*               pNumberOfCases);
+
 // Associates a native call site, identified by its offset in the native code stream, with
 // the signature information and method handle the JIT used to lay out the call site. If
 // the call site has no signature information (e.g. a helper call) or has no method handle

src/coreclr/src/ToolBox/superpmi/superpmi-shared/lwmlist.h

@@ -100,6 +100,7 @@ LWM(GetJitFlags, DWORD, DD)
 LWM(GetJitTimeLogFilename, DWORD, DWORD)
 LWM(GetJustMyCodeHandle, DWORDLONG, DLDL)
 LWM(GetLazyStringLiteralHelper, DWORDLONG, DWORD)
+LWM(GetLikelyClass, Agnostic_GetLikelyClass, Agnostic_GetLikelyClassResult)
 LWM(GetLocationOfThisType, DWORDLONG, Agnostic_CORINFO_LOOKUP_KIND)
 LWM(GetMethodAttribs, DWORDLONG, DWORD)
 LWM(GetMethodClass, DWORDLONG, DWORDLONG)

src/coreclr/src/ToolBox/superpmi/superpmi-shared/methodcontext.cpp

@@ -5265,6 +5265,48 @@ HRESULT MethodContext::repGetMethodBlockCounts(CORINFO_METHOD_HANDLE        ftnH
     return result;
 }
 
+void MethodContext::recGetLikelyClass(CORINFO_METHOD_HANDLE ftnHnd, CORINFO_CLASS_HANDLE baseHnd, UINT32 ilOffset, CORINFO_CLASS_HANDLE result, UINT32* pLikelihood, UINT32* pNumberOfClasses)
+{
+    if (GetLikelyClass == nullptr)
+        GetLikelyClass = new LightWeightMap<Agnostic_GetLikelyClass, Agnostic_GetLikelyClassResult>();
+
+    Agnostic_GetLikelyClass key;
+    ZeroMemory(&key, sizeof(Agnostic_GetLikelyClass));
+
+    key.ftnHnd = (DWORDLONG) ftnHnd;
+    key.baseHnd = (DWORDLONG) baseHnd;
+    key.ilOffset = (DWORD) ilOffset;
+
+    Agnostic_GetLikelyClassResult value;
+    ZeroMemory(&value, sizeof(Agnostic_GetLikelyClassResult));
+    value.classHnd = (DWORDLONG) result;
+    value.likelihood = *pLikelihood;
+    value.numberOfClasses = *pNumberOfClasses;
+
+    GetLikelyClass->Add(key, value);
+    DEBUG_REC(dmpGetLikelyClass(key, value));
+}
+void MethodContext::dmpGetLikelyClass(const Agnostic_GetLikelyClass& key, const Agnostic_GetLikelyClassResult& value)
+{
+    printf("GetLikelyClass key ftn-%016llX base-%016llX il-%u, class-%016llX likelihood-%u numberOfClasses-%u", 
+        key.ftnHnd, key.baseHnd, key.ilOffset, value.classHnd, value.likelihood, value.numberOfClasses);
+}
+CORINFO_CLASS_HANDLE MethodContext::repGetLikelyClass(CORINFO_METHOD_HANDLE ftnHnd, CORINFO_CLASS_HANDLE baseHnd, UINT32 ilOffset, UINT32* pLikelihood, UINT32* pNumberOfClasses)
+{
+    Agnostic_GetLikelyClass key;
+    ZeroMemory(&key, sizeof(Agnostic_GetLikelyClass));
+    key.ftnHnd = (DWORDLONG) ftnHnd;
+    key.baseHnd = (DWORDLONG) baseHnd;
+    key.ilOffset = (DWORD) ilOffset;
+
+    Agnostic_GetLikelyClassResult value = GetLikelyClass->Get(key);
+    DEBUG_REP(dmpGetLikelyClass(key, value));
+
+    *pLikelihood = value.likelihood;
+    *pNumberOfClasses = value.numberOfClasses;
+    return (CORINFO_CLASS_HANDLE) value.classHnd;
+}
+
 void MethodContext::recMergeClasses(CORINFO_CLASS_HANDLE cls1, CORINFO_CLASS_HANDLE cls2, CORINFO_CLASS_HANDLE result)
 {
     if (MergeClasses == nullptr)

src/coreclr/src/ToolBox/superpmi/superpmi-shared/methodcontext.h

@@ -435,6 +435,21 @@ class MethodContext
         DWORD numRuns;
         DWORD result;
     };
+
+    struct Agnostic_GetLikelyClass
+    {
+        DWORDLONG ftnHnd;
+        DWORDLONG baseHnd;
+        DWORD     ilOffset;
+    };
+
+    struct Agnostic_GetLikelyClassResult
+    {
+        DWORDLONG classHnd;
+        DWORD     likelihood;
+        DWORD     numberOfClasses;
+    };
+
     struct Agnostic_GetProfilingHandle
     {
         DWORD     bHookFunction;
@@ -1193,6 +1208,10 @@ class MethodContext
                                     ICorJitInfo::BlockCounts**   pBlockCounts,
                                     UINT32 *                     pNumRuns);
 
+    void recGetLikelyClass(CORINFO_METHOD_HANDLE ftnHnd, CORINFO_CLASS_HANDLE  baseHnd, UINT32 ilOffset, CORINFO_CLASS_HANDLE classHnd, UINT32* pLikelihood, UINT32* pNumberOfClasses);
+    void dmpGetLikelyClass(const Agnostic_GetLikelyClass& key, const Agnostic_GetLikelyClassResult& value);
+    CORINFO_CLASS_HANDLE repGetLikelyClass(CORINFO_METHOD_HANDLE ftnHnd, CORINFO_CLASS_HANDLE  baseHnd, UINT32 ilOffset, UINT32* pLikelihood, UINT32* pNumberOfClasses);
+
     void recMergeClasses(CORINFO_CLASS_HANDLE cls1, CORINFO_CLASS_HANDLE cls2, CORINFO_CLASS_HANDLE result);
     void dmpMergeClasses(DLDL key, DWORDLONG value);
     CORINFO_CLASS_HANDLE repMergeClasses(CORINFO_CLASS_HANDLE cls1, CORINFO_CLASS_HANDLE cls2);
@@ -1359,7 +1378,7 @@ class MethodContext
 };
 
 // ********************* Please keep this up-to-date to ease adding more ***************
-// Highest packet number: 181
+// Highest packet number: 182
 // *************************************************************************************
 enum mcPackets
 {
@@ -1458,6 +1477,7 @@ enum mcPackets
     Packet_GetJitFlags                                   = 154, // Added 2/3/2016
     Packet_GetJitTimeLogFilename                         = 67,
     Packet_GetJustMyCodeHandle                           = 68,
+    Packet_GetLikelyClass                                = 182, // Added 9/27/2020
     Packet_GetLocationOfThisType                         = 69,
     Packet_GetMethodAttribs                              = 70,
     Packet_GetMethodClass                                = 71,

src/coreclr/src/ToolBox/superpmi/superpmi-shim-collector/icorjitinfo.cpp

@@ -2049,6 +2049,21 @@ HRESULT interceptor_ICJI::getMethodBlockCounts(CORINFO_METHOD_HANDLE ftnHnd,
     return temp;
 }
 
+// Get the likely implementing class for a virtual call or interface call made by ftnHnd
+// at the indicated IL offset. baseHnd is the interface class or base class for the method
+// being called. 
+CORINFO_CLASS_HANDLE interceptor_ICJI::getLikelyClass(CORINFO_METHOD_HANDLE ftnHnd,
+                                                      CORINFO_CLASS_HANDLE  baseHnd,
+                                                      UINT32                ilOffset,
+                                                      UINT32*               pLikelihood,
+                                                      UINT32*               pNumberOfClasses)
+{
+    mc->cr->AddCall("getLikelyClass");
+    CORINFO_CLASS_HANDLE result = original_ICorJitInfo->getLikelyClass(ftnHnd, baseHnd, ilOffset, pLikelihood, pNumberOfClasses);
+    mc->recGetLikelyClass(ftnHnd, baseHnd, ilOffset, result, pLikelihood, pNumberOfClasses);
+    return result;
+}
+
 // Associates a native call site, identified by its offset in the native code stream, with
 // the signature information and method handle the JIT used to lay out the call site. If
 // the call site has no signature information (e.g. a helper call) or has no method handle

src/coreclr/src/ToolBox/superpmi/superpmi-shim-counter/icorjitinfo.cpp

@@ -1623,6 +1623,19 @@ HRESULT interceptor_ICJI::getMethodBlockCounts(CORINFO_METHOD_HANDLE ftnHnd,
     return original_ICorJitInfo->getMethodBlockCounts(ftnHnd, pCount, pBlockCounts, pNumRuns);
 }
 
+// Get the likely implementing class for a virtual call or interface call made by ftnHnd
+// at the indicated IL offset. baseHnd is the interface class or base class for the method
+// being called. 
+CORINFO_CLASS_HANDLE interceptor_ICJI::getLikelyClass(CORINFO_METHOD_HANDLE ftnHnd,
+                                                      CORINFO_CLASS_HANDLE  baseHnd,
+                                                      UINT32                ilOffset,
+                                                      UINT32*               pLikelihood,
+                                                      UINT32*               pNumberOfClasses)
+{
+    mcs->AddCall("getLikelyClass");
+    return original_ICorJitInfo->getLikelyClass(ftnHnd, baseHnd, ilOffset, pLikelihood, pNumberOfClasses);
+}
+
 // Associates a native call site, identified by its offset in the native code stream, with
 // the signature information and method handle the JIT used to lay out the call site. If
 // the call site has no signature information (e.g. a helper call) or has no method handle

src/coreclr/src/ToolBox/superpmi/superpmi-shim-simple/icorjitinfo.cpp

@@ -1441,6 +1441,18 @@ HRESULT interceptor_ICJI::getMethodBlockCounts(CORINFO_METHOD_HANDLE ftnHnd,
     return original_ICorJitInfo->getMethodBlockCounts(ftnHnd, pCount, pBlockCounts, pNumRuns);
 }
 
+// Get the likely implementing class for a virtual call or interface call made by ftnHnd
+// at the indicated IL offset. baseHnd is the interface class or base class for the method
+// being called. 
+CORINFO_CLASS_HANDLE interceptor_ICJI::getLikelyClass(CORINFO_METHOD_HANDLE ftnHnd,
+                                                      CORINFO_CLASS_HANDLE  baseHnd,
+                                                      UINT32                ilOffset,
+                                                      UINT32*               pLikelihood,
+                                                      UINT32*               pNumberOfClasses)
+{
+    return original_ICorJitInfo->getLikelyClass(ftnHnd, baseHnd, ilOffset, pLikelihood, pNumberOfClasses);
+}
+
 // Associates a native call site, identified by its offset in the native code stream, with
 // the signature information and method handle the JIT used to lay out the call site. If
 // the call site has no signature information (e.g. a helper call) or has no method handle

src/coreclr/src/ToolBox/superpmi/superpmi/icorjitinfo.cpp

@@ -1805,6 +1805,19 @@ HRESULT MyICJI::getMethodBlockCounts(CORINFO_METHOD_HANDLE ftnHnd,
     return jitInstance->mc->repGetMethodBlockCounts(ftnHnd, pCount, pBlockCounts, pNumRuns);
 }
 
+// Get the likely implementing class for a virtual call or interface call made by ftnHnd
+// at the indicated IL offset. baseHnd is the interface class or base class for the method
+// being called. 
+CORINFO_CLASS_HANDLE MyICJI::getLikelyClass(CORINFO_METHOD_HANDLE ftnHnd,
+                                            CORINFO_CLASS_HANDLE  baseHnd,
+                                            UINT32                ilOffset,
+                                            UINT32*               pLikelihood,
+                                            UINT32*               pNumberOfClasses)
+{
+    jitInstance->mc->cr->AddCall("getLikelyClass");
+    return jitInstance->mc->repGetLikelyClass(ftnHnd, baseHnd, ilOffset, pLikelihood, pNumberOfClasses);
+}
+
 // Associates a native call site, identified by its offset in the native code stream, with
 // the signature information and method handle the JIT used to lay out the call site. If
 // the call site has no signature information (e.g. a helper call) or has no method handle

src/coreclr/src/inc/corinfo.h

@@ -208,11 +208,11 @@ TODO: Talk about initializing strutures before use
 //
 //////////////////////////////////////////////////////////////////////////////////////////////////////////
 
-constexpr GUID JITEEVersionIdentifier = { /* 8031aa05-4568-40fc-a0d2-d971d8edba16 */
-    0x8031aa05,
-    0x4568,
-    0x40fc,
-    {0xa0, 0xd2, 0xd9, 0x71, 0xd8, 0xed, 0xba, 0x16}
+constexpr GUID JITEEVersionIdentifier = { /* 0d235fe4-65a1-487a-8553-c845496da901 */
+    0x0d235fe4,
+    0x65a1,
+    0x487a,
+    {0x85, 0x53, 0xc8, 0x45, 0x49, 0x6d, 0xa9, 0x01}
 };
 
 //////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -623,6 +623,7 @@ enum CorInfoHelpFunc
     CORINFO_HELP_STACK_PROBE,               // Probes each page of the allocated stack frame
 
     CORINFO_HELP_PATCHPOINT,                // Notify runtime that code has reached a patchpoint
+    CORINFO_HELP_CLASSPROFILE,              // Update class profile for a call site
 
     CORINFO_HELP_COUNT,
 };

src/coreclr/src/inc/corjit.h

@@ -251,6 +251,34 @@ class ICorJitInfo : public ICorDynamicInfo
         UINT32 ExecutionCount;
     };
 
+    // Data structure for a single class probe.
+    //
+    // ILOffset is the IL offset in the method for the call site being probed.
+    // Currently it must be ORed with CLASS_FLAG and (for interface calls)
+    // INTERFACE_FLAG.
+    //
+    // Count is the number of times a call was made at that call site.
+    //
+    // SIZE is the number of entries in the table.
+    //
+    // SAMPLE_INTERVAL must be >= SIZE. SAMPLE_INTERVAL / SIZE
+    // gives the average number of calls between table updates.
+    // 
+    struct ClassProfile
+    {
+        enum { 
+            SIZE = 8, 
+            SAMPLE_INTERVAL = 32, 
+            CLASS_FLAG     = 0x80000000, 
+            INTERFACE_FLAG = 0x40000000,
+            OFFSET_MASK    = 0x3FFFFFFF
+        };
+
+        UINT32 ILOffset;
+        UINT32 Count;
+        CORINFO_CLASS_HANDLE ClassTable[SIZE];
+    };
+
     // allocate a basic block profile buffer where execution counts will be stored
     // for jitted basic blocks.
     virtual HRESULT allocMethodBlockCounts (
@@ -267,6 +295,24 @@ class ICorJitInfo : public ICorDynamicInfo
             UINT32 *              pNumRuns         // pointer to the total number of profile scenarios run
             ) = 0;
 
+    // Get the likely implementing class for a virtual call or interface call made by ftnHnd
+    // at the indicated IL offset. baseHnd is the interface class or base class for the method
+    // being called. May returns NULL.
+    // 
+    // pLikelihood is the estimated percent chance that the class at runtime is the class
+    // returned by this method. A well-estimated monomorphic call site will return a likelihood
+    // of 100.
+    // 
+    // pNumberOfClasses is the estimated number of different classes seen at the site.
+    // A well-estimated monomorphic call site will return 1.
+    virtual CORINFO_CLASS_HANDLE getLikelyClass(
+            CORINFO_METHOD_HANDLE ftnHnd,
+            CORINFO_CLASS_HANDLE  baseHnd,
+            UINT32                ilOffset,
+            UINT32 *              pLikelihood,      // OUT, estimated likelihood of the class (0...100)
+            UINT32 *              pNumberOfClasses  // OUT, estimated number of possible classes
+            ) = 0;
+
     // Associates a native call site, identified by its offset in the native code stream, with
     // the signature information and method handle the JIT used to lay out the call site. If
     // the call site has no signature information (e.g. a helper call) or has no method handle

src/coreclr/src/inc/jithelpers.h

@@ -355,6 +355,7 @@
 #endif
 
     JITHELPER(CORINFO_HELP_PATCHPOINT, JIT_Patchpoint, CORINFO_HELP_SIG_REG_ONLY)
+    JITHELPER(CORINFO_HELP_CLASSPROFILE, JIT_ClassProfile, CORINFO_HELP_SIG_REG_ONLY)
 
 #undef JITHELPER
 #undef DYNAMICJITHELPER

src/coreclr/src/jit/block.h

@@ -448,6 +448,7 @@ struct BasicBlock : private LIR::Range
 
 #define BBF_BACKWARD_JUMP_TARGET           MAKE_BBFLAG(36) // Block is a target of a backward jump
 #define BBF_PATCHPOINT                     MAKE_BBFLAG(37) // Block is a patchpoint
+#define BBF_HAS_CLASS_PROFILE              MAKE_BBFLAG(38) // BB contains a call needing a class profile
 
 // clang-format on
 
@@ -492,7 +493,7 @@ struct BasicBlock : private LIR::Range
 #define BBF_SPLIT_GAINED                                                                                               \
     (BBF_DONT_REMOVE | BBF_HAS_LABEL | BBF_HAS_JMP | BBF_BACKWARD_JUMP | BBF_HAS_IDX_LEN | BBF_HAS_NEWARRAY |          \
      BBF_PROF_WEIGHT | BBF_HAS_NEWOBJ | BBF_KEEP_BBJ_ALWAYS | BBF_CLONED_FINALLY_END | BBF_HAS_NULLCHECK |             \
-     BBF_HAS_VTABREF)
+     BBF_HAS_VTABREF | BBF_HAS_CLASS_PROFILE)
 
 #ifndef __GNUC__ // GCC doesn't like C_ASSERT at global scope
     static_assert_no_msg((BBF_SPLIT_NONEXIST & BBF_SPLIT_LOST) == 0);

src/coreclr/src/jit/compiler.cpp

@@ -5982,6 +5982,7 @@ int Compiler::compCompileHelper(CORINFO_MODULE_HANDLE classPtr,
     info.compNativeCodeSize    = 0;
     info.compTotalHotCodeSize  = 0;
     info.compTotalColdCodeSize = 0;
+    info.compClassProbeCount   = 0;
 
     compHasBackwardJump = false;
 

src/coreclr/src/jit/compiler.h

@@ -3779,7 +3779,8 @@ class Compiler
                              CORINFO_CONTEXT_HANDLE* contextHandle,
                              CORINFO_CONTEXT_HANDLE* exactContextHandle,
                              bool                    isLateDevirtualization,
-                             bool                    isExplicitTailCall);
+                             bool                    isExplicitTailCall,
+                             IL_OFFSETX              ilOffset = BAD_IL_OFFSET);
 
     //=========================================================================
     //                          PROTECTED
@@ -6746,7 +6747,8 @@ class Compiler
                                              CORINFO_METHOD_HANDLE methodHandle,
                                              CORINFO_CLASS_HANDLE  classHandle,
                                              unsigned              methodAttr,
-                                             unsigned              classAttr);
+                                             unsigned              classAttr,
+                                             unsigned              likelihood);
 
     bool doesMethodHaveExpRuntimeLookup()
     {
@@ -9311,6 +9313,10 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 #define CPU_ARM64 0x0400 // The generic ARM64 CPU
 
         unsigned genCPU; // What CPU are we running on
+
+        // Number of class profile probes in this method
+        unsigned compClassProbeCount;
+
     } info;
 
     // Returns true if the method being compiled returns a non-void and non-struct value.