Handle more than 64 registers - Part 4 (#102921)

* Add high field

* Do not use `const regMaskTP&` as parameter

* Revert "Do not use `const regMaskTP&` as parameter"

This reverts commit d53b620.

By not passing `regMaskTP` using constant reference, there is a small cost
we have to pay:

Without constant reference:
Overall: 1.80%
MinOpts: 2.05%
FullOpts: 1.62%

With constant reference:
Overall: 1.74%
MinOpts: 1.94%
FullOpts: 1.6%

* Furture reduce the TP cost

Moved the *= operators as instance of `regMaskTP` so the `.low`
private field can directly be manipulated instead of converting
the `64-bit` value in `regMaskTP` before doing any operation.

Overall: 0.74%
MinOpts: 0.82%
FullOpts: 0.68%

* jit fornat

* Add high to all platforms

* use genSingleTypeRegMask()

* Make high only to arm64

* Use HAS_MORE_THAN_64_REGISTERS macro

* Add regTypeTag in register*.h files

* change the structure of regMaskTP for performance

* AddRegNumInMask/AddRegNum

* Use AddRegNumInMask() and AddRegNum()

* RemoveRegNumFromMask() and RemoveRegNum() and consume them

* Use IsEmpty() and IsNonEmpty()

* IsRegNumInMask() and IsRegNumPresent()

* AddGprRegs()

* AddRegsetForType()

* RemoveRegsetForType()

* GetRegSetForType() defintion

* operators() that operate on regNum

* misc cleanup

* fix some other build failures

* jit format

* Add operator[]

* Move mapTypeToRegTypeIndex() and mapRegNumtoRegTypeIndex() in compiler.hpp

* Move registerType in Referencable, add getRegisterType()

* Add various variants of genRegNumFromMask() and use them

* Change some more methods to return SingleTypeRegMask

* Initialize regMaskTP in genRegMask()

* bug fix

* Add assert in genFindLowestBit() and genMaxOneBit() to be only applicable for gpr/float

* genRegNumFromMask() to take SingleTypeRegSet

* jit format

* fix another assert

* Rename getRegMask to getSingleTypeRegMask

This will return `SingleTypeRegMask` and will save some cost of creating
`regMaskTP` unnecessarilyt.

* Move HAS_MORE_THAN_64_REGISTERS related macros

* skip unnecessary initialization

* fix some places where implicit conversion from `SingleTypeRegSet -> regMaskTP` was happening

* fix a bug for populating availableRegs

* Simplify the updating of high field

It also reduces the TP cost. The original thinking was branches can
be expensive and hence condition-less code was added, but we end up
with more instructions with condition-less. Additionally, branch predicators
should be better in making sure that mostly we would take the gpr/float code
path and rarely take the predicate registers code path. Also, we do PGO builds
which should feed that information and make branch code cheaper in TP cost wise.

* Revert "Add regTypeTag in register*.h files"

This reverts commit 4fe1caa.

* misc cleanup

* jit format

* fix the linux build error

* Revert "fix the linux build error"

This reverts commit 0b67c48.

* fix linux error properly

* fix linux/arm bug

* review feedback

src/coreclr/jit/codegencommon.cpp

@@ -584,7 +584,7 @@ void CodeGenInterface::genUpdateLife(VARSET_VALARG_TP newLife)
 // inline
 regMaskTP CodeGenInterface::genGetRegMask(const LclVarDsc* varDsc)
 {
-    regMaskTP regMask = RBM_NONE;
+    regMaskTP regMask;
 
     assert(varDsc->lvIsInReg());
 

src/coreclr/jit/compiler.h

@@ -1015,9 +1015,10 @@ class LclVarDsc
 
     regMaskTP lvRegMask() const
     {
-        regMaskTP regMask = RBM_NONE;
         if (GetRegNum() != REG_STK)
         {
+            regMaskTP regMask;
+
             if (varTypeUsesFloatReg(this))
             {
                 regMask = genRegMaskFloat(GetRegNum() ARM_ARG(TypeGet()));
@@ -1032,8 +1033,12 @@ class LclVarDsc
 
                 regMask = genRegMask(GetRegNum());
             }
+            return regMask;
+        }
+        else
+        {
+            return RBM_NONE;
         }
-        return regMask;
     }
 
     //-----------------------------------------------------------------------------

src/coreclr/jit/compiler.hpp

@@ -101,6 +101,11 @@ inline bool genExactlyOneBit(T value)
 
 inline regMaskTP genFindLowestBit(regMaskTP value)
 {
+#ifdef HAS_MORE_THAN_64_REGISTERS
+    // If we ever need to use this method for predicate
+    // registers, then handle it.
+    assert(value.getHigh() == RBM_NONE);
+#endif
     return regMaskTP(genFindLowestBit(value.getLow()));
 }
 
@@ -111,19 +116,14 @@ inline regMaskTP genFindLowestBit(regMaskTP value)
 
 inline bool genMaxOneBit(regMaskTP value)
 {
+#ifdef HAS_MORE_THAN_64_REGISTERS
+    // If we ever need to use this method for predicate
+    // registers, then handle it.
+    assert(value.getHigh() == RBM_NONE);
+#endif
     return genMaxOneBit(value.getLow());
 }
 
-/*****************************************************************************
- *
- *  Return true if the given value has exactly one bit set.
- */
-
-inline bool genExactlyOneBit(regMaskTP value)
-{
-    return genExactlyOneBit(value.getLow());
-}
-
 /*****************************************************************************
  *
  *  Given a value that has exactly one bit set, return the position of that
@@ -169,7 +169,7 @@ inline unsigned uhi32(uint64_t value)
 
 inline unsigned genCountBits(regMaskTP mask)
 {
-    return BitOperations::PopCount(mask.getLow());
+    return PopCount(mask);
 }
 
 /*****************************************************************************
@@ -933,16 +933,115 @@ inline unsigned Compiler::funGetFuncIdx(BasicBlock* block)
 // Assumptions:
 //    The mask contains one and only one register.
 
-inline regNumber genRegNumFromMask(regMaskTP mask)
+inline regNumber genRegNumFromMask(const SingleTypeRegSet& mask)
+{
+    assert(mask != RBM_NONE); // Must have one bit set, so can't have a mask of zero
+
+    /* Convert the mask to a register number */
+
+    regNumber regNum = (regNumber)genLog2(mask);
+
+    /* Make sure we got it right */
+    assert(genSingleTypeRegMask(regNum) == mask);
+
+    return regNum;
+}
+
+//------------------------------------------------------------------------------
+// genRegNumFromMask : Maps a single register mask having gpr/float to a register number.
+//          If the mask can contain predicate register, use genRegNumFromMask(reg, type)
+//
+// Arguments:
+//    mask - the register mask
+//
+// Return Value:
+//    The number of the register contained in the mask.
+//
+// Assumptions:
+//    The mask contains one and only one register.
+
+inline regNumber genRegNumFromMask(const regMaskTP& mask)
 {
+#ifdef HAS_MORE_THAN_64_REGISTERS
+    // This method is only used for gpr/float
+    assert(mask.getHigh() == RBM_NONE);
+#endif
+
     assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
 
     /* Convert the mask to a register number */
 
     regNumber regNum = (regNumber)genLog2(mask.getLow());
 
     /* Make sure we got it right */
-    assert(genRegMask(regNum) == mask.getLow());
+    assert(genRegMask(regNum).getLow() == mask.getLow());
+
+    return regNum;
+}
+
+//------------------------------------------------------------------------------
+// genRegNumFromMask : Maps a single register mask to a register number.
+//
+// Arguments:
+//    mask - the register mask
+//    type - The
+//
+// Return Value:
+//    The number of the register contained in the mask.
+//
+// Assumptions:
+//    The mask contains one and only one register.
+
+inline regNumber genRegNumFromMask(SingleTypeRegSet mask, var_types type)
+{
+    regNumber regNum = genRegNumFromMask(mask);
+
+#ifdef HAS_MORE_THAN_64_REGISTERS
+    if (varTypeIsMask(type))
+    {
+        regNum = (regNumber)(64 + regNum);
+    }
+#endif
+    return regNum;
+}
+
+//------------------------------------------------------------------------------
+// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
+//
+// Arguments:
+//    mask               - the register mask
+//
+// Return Value:
+//    The number of the first register contained in the mask.
+//
+// TODO: check if const regMaskTP& matter or should we pass by value
+inline regNumber genFirstRegNumFromMask(const regMaskTP& mask)
+{
+    assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+
+    /* Convert the mask to a register number */
+
+    regNumber regNum = (regNumber)BitScanForward(mask);
+
+    return regNum;
+}
+
+//------------------------------------------------------------------------------
+// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
+//
+// Arguments:
+//    mask               - the register mask
+//
+// Return Value:
+//    The number of the first register contained in the mask.
+//
+inline regNumber genFirstRegNumFromMask(SingleTypeRegSet mask)
+{
+    assert(mask != RBM_NONE); // Must have one bit set, so can't have a mask of zero
+
+    /* Convert the mask to a register number */
+
+    regNumber regNum = (regNumber)BitOperations::BitScanForward(mask);
 
     return regNum;
 }
@@ -966,28 +1065,31 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskTP& mask)
 
     regNumber regNum = (regNumber)BitScanForward(mask);
 
-    mask ^= genRegMask(regNum);
+    mask.RemoveRegNumFromMask(regNum);
 
     return regNum;
 }
 
 //------------------------------------------------------------------------------
-// genFirstRegNumFromMask : Maps first bit set in the register mask to a register number.
-//
+// genFirstRegNumFromMaskAndToggle : Maps first bit set in the register mask to a
+//          register number and also toggle the bit in the `mask`.
 // Arguments:
 //    mask               - the register mask
 //
 // Return Value:
-//    The number of the first register contained in the mask.
+//    The number of the first register contained in the mask and updates the `mask` to toggle
+//    the bit.
 //
 
-inline regNumber genFirstRegNumFromMask(regMaskTP mask)
+inline regNumber genFirstRegNumFromMaskAndToggle(SingleTypeRegSet& mask)
 {
-    assert(mask.IsNonEmpty()); // Must have one bit set, so can't have a mask of zero
+    assert(mask != RBM_NONE); // Must have one bit set, so can't have a mask of zero
 
     /* Convert the mask to a register number */
 
-    regNumber regNum = (regNumber)BitScanForward(mask);
+    regNumber regNum = (regNumber)BitOperations::BitScanForward(mask);
+
+    mask ^= genSingleTypeRegMask(regNum);
 
     return regNum;
 }