SPMI: Add and utilize number of contexts with diffs

src/coreclr/jit/forwardsub.cpp

@@ -116,6 +116,12 @@ PhaseStatus Compiler::fgForwardSub()
     }
 #endif
 
+    CLRRandom rng;
+    rng.Init(info.compILCodeSize ^ info.compILlocalsCount ^ 0x12345678);
+    if (rng.Next(10) == 0)
+    {
+        return PhaseStatus::MODIFIED_NOTHING;
+    }
     bool changed = false;
 
     for (BasicBlock* const block : Blocks())

src/coreclr/scripts/superpmi.py

@@ -1893,17 +1893,26 @@ def sum_diff(row, col):
                         html_color(base_color, "{:,d}".format(missing_base_contexts)),
                         html_color(diff_color, "{:,d}".format(missing_diff_contexts))))
 
-                if any(has_diff for (_, _, _, has_diff, _) in asm_diffs):
+                def has_diffs(row):
+                    return int(row["Contexts with diffs"]) > 0
+
+                # Exclude entire diffs section?
+                if any(has_diffs(diff_metrics["Overall"]) for (_, _, diff_metrics, _, _) in asm_diffs):
                     def write_pivot_section(row):
+                        # Exclude this particular Overall/MinOpts/FullOpts table?
+                        if not any(has_diffs(diff_metrics[row]) for (_, _, diff_metrics, _, _) in asm_diffs):
+                            return
+
                         write_fh.write("\n<details>\n")
                         sum_base = sum(int(base_metrics[row]["Diffed code bytes"]) for (_, base_metrics, _, _, _) in asm_diffs)
                         sum_diff = sum(int(diff_metrics[row]["Diffed code bytes"]) for (_, _, diff_metrics, _, _) in asm_diffs)
 
                         write_fh.write("<summary>{} ({} bytes)</summary>\n\n".format(row, format_delta(sum_base, sum_diff)))
                         write_fh.write("|Collection|Base size (bytes)|Diff size (bytes)|\n")
                         write_fh.write("|---|--:|--:|\n")
-                        for (mch_file, base_metrics, diff_metrics, has_diffs, _) in asm_diffs:
-                            if not has_diffs:
+                        for (mch_file, base_metrics, diff_metrics, _, _) in asm_diffs:
+                            # Exclude this particular row?
+                            if not has_diffs(diff_metrics[row]):
                                 continue
 
                             write_fh.write("|{}|{:,d}|{}|\n".format(
@@ -1925,14 +1934,15 @@ def write_pivot_section(row):
                 write_fh.write("\n<details>\n")
                 write_fh.write("<summary>Details</summary>\n\n")
 
-                write_fh.write("|Collection|Diffed contexts|MinOpts|FullOpts|Missed, base|Missed, diff|\n")
-                write_fh.write("|---|--:|--:|--:|--:|--:|\n")
+                write_fh.write("|Collection|Diffed contexts|MinOpts|FullOpts|Contexts with diffs|Missed, base|Missed, diff|\n")
+                write_fh.write("|---|--:|--:|--:|--:|--:|--:|\n")
                 for (mch_file, base_metrics, diff_metrics, has_diffs, jit_analyze_summary_file) in asm_diffs:
-                    write_fh.write("|{}|{:,d}|{:,d}|{:,d}|{:,d}|{:,d}|\n".format(
+                    write_fh.write("|{}|{:,d}|{:,d}|{:,d}|{:,d}|{:,d}|{:,d}|\n".format(
                         mch_file,
                         int(diff_metrics["Overall"]["Successful compiles"]),
                         int(diff_metrics["MinOpts"]["Successful compiles"]),
                         int(diff_metrics["FullOpts"]["Successful compiles"]),
+                        int(diff_metrics["Overall"]["Contexts with diffs"]),
                         int(base_metrics["Overall"]["Missing compiles"]),
                         int(diff_metrics["Overall"]["Missing compiles"])))
 
@@ -2159,11 +2169,8 @@ def replay_with_throughput_diff(self):
                 # instruction count and all collections.
                 def is_significant_pct(base, diff):
                     return round((diff - base) / base * 100, 2) != 0
-                def is_significant(base, diff):
-                    def check(col):
-                        return is_significant_pct(int(base[col]["Diff executed instructions"]), int(diff[col]["Diff executed instructions"]))
-
-                    return check("Overall") or check("MinOpts") or check("FullOpts")
+                def is_significant(row, base, diff):
+                    return is_significant_pct(int(base[row]["Diff executed instructions"]), int(diff[row]["Diff executed instructions"]))
                 def format_pct(base_instructions, diff_instructions):
                     plus_if_positive = "+" if diff_instructions > base_instructions else ""
                     text = "{}{:.2f}%".format(plus_if_positive, (diff_instructions - base_instructions) / base_instructions * 100)
@@ -2173,24 +2180,23 @@ def format_pct(base_instructions, diff_instructions):
 
                     return text
 
-                significant_diffs = {}
-                for mch_file, base, diff in tp_diffs:
-                    significant_diffs[mch_file] = is_significant(base, diff)
+                if any(is_significant(row, base, diff) for row in ["Overall", "MinOpts", "FullOpts"] for (_, base, diff) in tp_diffs):
+                    def write_pivot_section(row):
+                        if not any(is_significant(row, base, diff) for (_, base, diff) in tp_diffs):
+                            return
 
-                if any(significant_diffs[mch_file] for (mch_file, _, _) in tp_diffs):
-                    def write_pivot_section(col):
                         write_fh.write("\n<details>\n")
-                        sum_base = sum(int(base_metrics[col]["Diff executed instructions"]) for (_, base_metrics, _) in tp_diffs)
-                        sum_diff = sum(int(diff_metrics[col]["Diff executed instructions"]) for (_, _, diff_metrics) in tp_diffs)
+                        sum_base = sum(int(base_metrics[row]["Diff executed instructions"]) for (_, base_metrics, _) in tp_diffs)
+                        sum_diff = sum(int(diff_metrics[row]["Diff executed instructions"]) for (_, _, diff_metrics) in tp_diffs)
 
-                        write_fh.write("<summary>{} ({})</summary>\n\n".format(col, format_pct(sum_base, sum_diff)))
+                        write_fh.write("<summary>{} ({})</summary>\n\n".format(row, format_pct(sum_base, sum_diff)))
                         write_fh.write("|Collection|PDIFF|\n")
                         write_fh.write("|---|--:|\n")
                         for mch_file, base, diff in tp_diffs:
-                            base_instructions = int(base[col]["Diff executed instructions"])
-                            diff_instructions = int(diff[col]["Diff executed instructions"])
+                            base_instructions = int(base[row]["Diff executed instructions"])
+                            diff_instructions = int(diff[row]["Diff executed instructions"])
 
-                            if significant_diffs[mch_file]:
+                            if is_significant(row, base, diff):
                                 write_fh.write("|{}|{}|\n".format(
                                     mch_file,
                                     format_pct(base_instructions, diff_instructions)))
@@ -2206,13 +2212,13 @@ def write_pivot_section(col):
 
                 write_fh.write("\n<details>\n")
                 write_fh.write("<summary>Details</summary>\n\n")
-                for (disp, col) in [("All", "Overall"), ("MinOpts", "MinOpts"), ("FullOpts", "FullOpts")]:
+                for (disp, row) in [("All", "Overall"), ("MinOpts", "MinOpts"), ("FullOpts", "FullOpts")]:
                     write_fh.write("{} contexts:\n\n".format(disp))
                     write_fh.write("|Collection|Base # instructions|Diff # instructions|PDIFF|\n")
                     write_fh.write("|---|--:|--:|--:|\n")
                     for mch_file, base, diff in tp_diffs:
-                        base_instructions = int(base[col]["Diff executed instructions"])
-                        diff_instructions = int(diff[col]["Diff executed instructions"])
+                        base_instructions = int(base[row]["Diff executed instructions"])
+                        diff_instructions = int(diff[row]["Diff executed instructions"])
                         write_fh.write("|{}|{:,d}|{:,d}|{}|\n".format(
                             mch_file, base_instructions, diff_instructions,
                             format_pct(base_instructions, diff_instructions)))

src/coreclr/tools/superpmi/superpmi/metricssummary.cpp

@@ -10,6 +10,7 @@ void MetricsSummary::AggregateFrom(const MetricsSummary& other)
     SuccessfulCompiles += other.SuccessfulCompiles;
     FailingCompiles += other.FailingCompiles;
     MissingCompiles += other.MissingCompiles;
+    NumContextsWithDiffs += other.NumContextsWithDiffs;
     NumCodeBytes += other.NumCodeBytes;
     NumDiffedCodeBytes += other.NumDiffedCodeBytes;
     NumExecutedInstructions += other.NumExecutedInstructions;
@@ -67,7 +68,7 @@ bool MetricsSummaries::SaveToFile(const char* path)
 
     if (!FilePrintf(
         file.get(),
-        "Successful compiles,Failing compiles,Missing compiles,"
+        "Successful compiles,Failing compiles,Missing compiles,Contexts with diffs,"
         "Code bytes,Diffed code bytes,Executed instructions,Diff executed instructions,Name\n"))
     {
         return false;
@@ -84,10 +85,11 @@ bool MetricsSummaries::WriteRow(HANDLE hFile, const char* name, const MetricsSum
     return
         FilePrintf(
             hFile,
-            "%d,%d,%d,%lld,%lld,%lld,%lld,%s\n",
+            "%d,%d,%d,%d,%lld,%lld,%lld,%lld,%s\n",
             summary.SuccessfulCompiles,
             summary.FailingCompiles,
             summary.MissingCompiles,
+            summary.NumContextsWithDiffs,
             summary.NumCodeBytes,
             summary.NumDiffedCodeBytes,
             summary.NumExecutedInstructions,
@@ -150,17 +152,18 @@ bool MetricsSummaries::LoadFromFile(const char* path, MetricsSummaries* metrics)
         int scanResult =
             sscanf_s(
                 line.data(),
-                "%d,%d,%d,%lld,%lld,%lld,%lld,%s",
+                "%d,%d,%d,%d,%lld,%lld,%lld,%lld,%s",
                 &summary.SuccessfulCompiles,
                 &summary.FailingCompiles,
                 &summary.MissingCompiles,
+                &summary.NumContextsWithDiffs,
                 &summary.NumCodeBytes,
                 &summary.NumDiffedCodeBytes,
                 &summary.NumExecutedInstructions,
                 &summary.NumDiffExecutedInstructions,
                 name, (unsigned)sizeof(name));
 
-        if (scanResult == 8)
+        if (scanResult == 9)
         {
             MetricsSummary* tarSummary = nullptr;
             if (strcmp(name, "Overall") == 0)

src/coreclr/tools/superpmi/superpmi/metricssummary.h

@@ -12,6 +12,8 @@ struct MetricsSummary
     int FailingCompiles = 0;
     // Number of methods that failed jitting due to missing SPMI data.
     int MissingCompiles = 0;
+    // Number of contexts that had any diff.
+    int NumContextsWithDiffs = 0;
     // Number of code bytes produced by the JIT for the successful compiles.
     long long NumCodeBytes = 0;
     // Number of code bytes that were diffed with the other compiler in diff mode.

src/coreclr/tools/superpmi/superpmi/superpmi.cpp

@@ -67,22 +67,22 @@ void SetSuperPmiTargetArchitecture(const char* targetArchitecture)
 // This function uses PAL_TRY, so it can't be in the a function that requires object unwinding. Extracting it out here
 // avoids compiler error.
 //
-void InvokeNearDiffer(NearDiffer*           nearDiffer,
-                      CommandLine::Options* o,
-                      MethodContext**       mc,
-                      CompileResult**       crl,
-                      int*                  matchCount,
-                      MethodContextReader** reader,
-                      MCList*               failingMCL,
-                      MCList*               diffMCL)
+static void InvokeNearDiffer(NearDiffer*           nearDiffer,
+                             CommandLine::Options* o,
+                             MethodContext**       mc,
+                             CompileResult**       crl,
+                             bool*                 hasDiff,
+                             MethodContextReader** reader,
+                             MCList*               failingMCL,
+                             MCList*               diffMCL)
 {
     struct Param : FilterSuperPMIExceptionsParam_CaptureException
     {
         NearDiffer*           nearDiffer;
         CommandLine::Options* o;
         MethodContext**       mc;
         CompileResult**       crl;
-        int*                  matchCount;
+        bool*                 hasDiff;
         MethodContextReader** reader;
         MCList*               failingMCL;
         MCList*               diffMCL;
@@ -91,19 +91,17 @@ void InvokeNearDiffer(NearDiffer*           nearDiffer,
     param.o          = o;
     param.mc         = mc;
     param.crl        = crl;
-    param.matchCount = matchCount;
+    param.hasDiff    = hasDiff;
     param.reader     = reader;
     param.failingMCL = failingMCL;
     param.diffMCL    = diffMCL;
+    *hasDiff = false;
 
     PAL_TRY(Param*, pParam, &param)
     {
-        if (pParam->nearDiffer->compare(*pParam->mc, *pParam->crl, (*pParam->mc)->cr))
-        {
-            (*pParam->matchCount)++;
-        }
-        else
+        if (!pParam->nearDiffer->compare(*pParam->mc, *pParam->crl, (*pParam->mc)->cr))
         {
+            *pParam->hasDiff = true;
             LogIssue(ISSUE_ASM_DIFF, "main method %d of size %d differs", (*pParam->reader)->GetMethodContextIndex(),
                      (*pParam->mc)->methodSize);
 
@@ -250,7 +248,6 @@ int __cdecl main(int argc, char* argv[])
 
     int loadedCount       = 0;
     int jittedCount       = 0;
-    int matchCount        = 0;
     int failToReplayCount = 0;
     int errorCount        = 0;
     int errorCount2       = 0;
@@ -289,9 +286,9 @@ int __cdecl main(int argc, char* argv[])
             st1.Stop();
             if (o.applyDiff)
             {
-                LogVerbose(" %2.1f%% - Loaded %d  Jitted %d  Matching %d  FailedCompile %d at %d per second",
-                           reader->PercentComplete(), loadedCount, jittedCount, matchCount, failToReplayCount,
-                           (int)((double)500 / st1.GetSeconds()));
+                LogVerbose(" %2.1f%% - Loaded %d  Jitted %d  Diffs %d  FailedCompile %d at %d per second",
+                           reader->PercentComplete(), loadedCount, jittedCount, totalBaseMetrics.Overall.NumContextsWithDiffs,
+                           failToReplayCount, (int)((double)500 / st1.GetSeconds()));
             }
             else
             {
@@ -555,7 +552,17 @@ int __cdecl main(int argc, char* argv[])
                 }
                 else
                 {
-                    InvokeNearDiffer(&nearDiffer, &o, &mc, &crl, &matchCount, &reader, &failingToReplayMCL, &diffMCL);
+                    bool hasDiff;
+                    InvokeNearDiffer(&nearDiffer, &o, &mc, &crl, &hasDiff, &reader, &failingToReplayMCL, &diffMCL);
+
+                    if (hasDiff)
+                    {
+                        totalBaseMetrics.Overall.NumContextsWithDiffs++;
+                        totalDiffMetrics.Overall.NumContextsWithDiffs++;
+
+                        totalBaseMetricsOpts.NumContextsWithDiffs++;
+                        totalDiffMetricsOpts.NumContextsWithDiffs++;
+                    }
 
                     totalBaseMetrics.Overall.NumDiffedCodeBytes += baseMetrics.NumCodeBytes;
                     totalDiffMetrics.Overall.NumDiffedCodeBytes += diffMetrics.NumCodeBytes;
@@ -638,7 +645,7 @@ int __cdecl main(int argc, char* argv[])
     if (o.applyDiff)
     {
         LogInfo(g_AsmDiffsSummaryFormatString, loadedCount, jittedCount, failToReplayCount, excludedCount,
-                missingCount, jittedCount - failToReplayCount - matchCount);
+                missingCount, totalDiffMetrics.Overall.NumContextsWithDiffs);
     }
     else
     {
@@ -679,7 +686,7 @@ int __cdecl main(int argc, char* argv[])
     {
         result = SpmiResult::Error;
     }
-    else if (o.applyDiff && (matchCount != jittedCount - missingCount))
+    else if (o.applyDiff && (totalDiffMetrics.Overall.NumContextsWithDiffs > 0))
     {
         result = SpmiResult::Diffs;
     }