address GT_ADDR(GT_HWIntrinsic) nodes

src/jit/compiler.h

@@ -7712,7 +7712,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
     {
         return isSIMDClass(pTypeInfo) || isHWSIMDClass(pTypeInfo);
     }
-    
+
     // Get the base (element) type and size in bytes for a SIMD type. Returns TYP_UNKNOWN
     // if it is not a SIMD type or is an unsupported base type.
     var_types getBaseTypeAndSizeOfSIMDType(CORINFO_CLASS_HANDLE typeHnd, unsigned* sizeBytes = nullptr);

src/jit/gentree.h

@@ -1619,6 +1619,11 @@ struct GenTree
     }
 #endif
 
+    bool OperIsSIMDorSimdHWintrinsic() const
+    {
+        return OperIsSIMD() || OperIsSimdHWIntrinsic();
+    }
+
     // This is here for cleaner GT_LONG #ifdefs.
     static bool OperIsLong(genTreeOps gtOper)
     {
@@ -4265,9 +4270,7 @@ inline bool GenTree::OperIsSimdHWIntrinsic() const
 {
     if (gtOper == GT_HWIntrinsic)
     {
-        // We cannot use AsHWIntrinsic() as it is not declared const
-        const GenTreeHWIntrinsic* hwIntrinsic = reinterpret_cast<const GenTreeHWIntrinsic*>(this);
-        return hwIntrinsic->isSIMD();
+        return this->AsHWIntrinsic()->isSIMD();
     }
     return false;
 }

src/jit/importer.cpp

@@ -1394,7 +1394,8 @@ GenTree* Compiler::impGetStructAddr(GenTree*             structVal,
         assert(structVal->gtObj.gtClass == structHnd);
         return (structVal->gtObj.Addr());
     }
-    else if (oper == GT_CALL || oper == GT_RET_EXPR || oper == GT_OBJ || oper == GT_MKREFANY)
+    else if (oper == GT_CALL || oper == GT_RET_EXPR || oper == GT_OBJ || oper == GT_MKREFANY ||
+             structVal->OperIsSimdHWIntrinsic())
     {
         unsigned tmpNum = lvaGrabTemp(true DEBUGARG("struct address for call/obj"));
 
@@ -1644,15 +1645,7 @@ GenTree* Compiler::impNormStructVal(GenTree*             structVal,
             }
 
 #ifdef FEATURE_SIMD
-            if (blockNode->OperGet() == GT_SIMD)
-            {
-                parent->gtOp.gtOp2 = impNormStructVal(blockNode, structHnd, curLevel, forceNormalization);
-                alreadyNormalized  = true;
-            }
-            else
-#endif
-#ifdef FEATURE_HW_INTRINSICS
-                if (blockNode->OperGet() == GT_HWIntrinsic && blockNode->AsHWIntrinsic()->isSIMD())
+            if (blockNode->OperIsSIMDorSimdHWintrinsic())
             {
                 parent->gtOp.gtOp2 = impNormStructVal(blockNode, structHnd, curLevel, forceNormalization);
                 alreadyNormalized  = true;

src/jit/morph.cpp

@@ -3741,7 +3741,7 @@ GenTreeCall* Compiler::fgMorphArgs(GenTreeCall* call)
                                 if (addr->OperGet() == GT_ADDR)
                                 {
                                     GenTree* addrChild = addr->gtOp.gtOp1;
-                                    if (addrChild->OperGet() == GT_SIMD)
+                                    if (addrChild->OperIsSIMDorSimdHWintrinsic())
                                     {
                                         needCpyBlk = true;
                                     }
@@ -9318,17 +9318,17 @@ GenTree* Compiler::fgMorphOneAsgBlockOp(GenTree* tree)
     unsigned             size;
     CORINFO_CLASS_HANDLE clsHnd = NO_CLASS_HANDLE;
 #ifdef FEATURE_SIMD
-    // importer introduces cpblk nodes with src = GT_ADDR(GT_SIMD)
+    // importer introduces cpblk nodes with src = GT_ADDR(GT_SIMD/GT_HWIntrinsic)
     // The SIMD type in question could be Vector2f which is 8-bytes in size.
     // The below check is to make sure that we don't turn that copyblk
     // into a assignment, since rationalizer logic will transform the
     // copyblk appropriately. Otherwise, the transformation made in this
     // routine will prevent rationalizer logic and we might end up with
-    // GT_ADDR(GT_SIMD) node post rationalization, leading to a noway assert
+    // GT_ADDR(GT_SIMD/GT_HWIntrinsic) node post rationalization, leading to a noway assert
     // in codegen.
     // TODO-1stClassStructs: This is here to preserve old behavior.
     // It should be eliminated.
-    if (src->OperGet() == GT_SIMD)
+    if (src->OperIsSIMDorSimdHWintrinsic())
     {
         return nullptr;
     }
@@ -10235,13 +10235,13 @@ GenTree* Compiler::fgMorphBlockOperand(GenTree* tree, var_types asgType, unsigne
         if (varTypeIsSIMD(asgType))
         {
             if ((indirTree != nullptr) && (lclNode == nullptr) && (indirTree->Addr()->OperGet() == GT_ADDR) &&
-                (indirTree->Addr()->gtGetOp1()->gtOper == GT_SIMD))
+                (indirTree->Addr()->gtGetOp1()->OperIsSIMDorSimdHWintrinsic()))
             {
                 assert(!isDest);
                 needsIndirection = false;
                 effectiveVal     = indirTree->Addr()->gtGetOp1();
             }
-            if (effectiveVal->OperIsSIMD() || effectiveVal->OperIsSimdHWIntrinsic())
+            if (effectiveVal->OperIsSIMDorSimdHWintrinsic())
             {
                 needsIndirection = false;
             }
@@ -11390,6 +11390,15 @@ GenTree* Compiler::getSIMDStructFromField(GenTree*   tree,
                 *simdSizeOut          = simdNode->gtSIMDSize;
                 *pBaseTypeOut         = simdNode->gtSIMDBaseType;
             }
+#ifdef FEATURE_HW_INTRINSICS
+            else if (obj->OperIsSimdHWIntrinsic())
+            {
+                ret                          = obj;
+                GenTreeHWIntrinsic* simdNode = obj->AsHWIntrinsic();
+                *simdSizeOut                 = simdNode->gtSIMDSize;
+                *pBaseTypeOut                = simdNode->gtSIMDBaseType;
+            }
+#endif // FEATURE_HW_INTRINSICS
         }
     }
     if (ret != nullptr)
@@ -19036,19 +19045,12 @@ Compiler::fgWalkResult Compiler::fgMarkAddrTakenLocalsPreCB(GenTree** pTree, fgW
 
         case GT_ADDR:
 #ifdef FEATURE_SIMD
-            if (tree->gtOp.gtOp1->OperGet() == GT_SIMD)
+            if (tree->gtOp.gtOp1->OperIsSIMDorSimdHWintrinsic())
             {
                 axcStack->Push(AXC_None);
             }
             else
 #endif // FEATURE_SIMD
-#ifdef FEATURE_HW_INTRINSICS
-                if (tree->gtOp.gtOp1->OperIsSimdHWIntrinsic())
-            {
-                axcStack->Push(AXC_None);
-            }
-            else
-#endif // FEATURE_HW_INTRINSICS
                 if (axc == AXC_Ind)
             {
                 axcStack->Push(AXC_None);

src/jit/rationalize.cpp

@@ -108,7 +108,7 @@ void Rationalizer::RewriteSIMDOperand(LIR::Use& use, bool keepBlk)
         addr->gtType = simdType;
         use.ReplaceWith(comp, addr);
     }
-    else if ((addr->OperGet() == GT_ADDR) && (addr->gtGetOp1()->OperGet() == GT_SIMD))
+    else if ((addr->OperGet() == GT_ADDR) && (addr->gtGetOp1()->OperIsSIMDorSimdHWintrinsic()))
     {
         // if we have GT_IND(GT_ADDR(GT_SIMD)), remove the GT_IND(GT_ADDR()), leaving just the GT_SIMD.
         BlockRange().Remove(tree);