Implement DivRem intrinsic for X86

src/coreclr/jit/compiler.h

@@ -3357,6 +3357,8 @@ class Compiler
                                                    NamedIntrinsic hwIntrinsicID);
     GenTreeHWIntrinsic* gtNewScalarHWIntrinsicNode(
         var_types type, GenTree* op1, GenTree* op2, GenTree* op3, NamedIntrinsic hwIntrinsicID);
+    GenTreeLclVar* gtNewLclvForMultiRegIntrinsicNode(GenTreeHWIntrinsic* intrinsicNode, CORINFO_SIG_INFO* sig);
+
     CORINFO_CLASS_HANDLE gtGetStructHandleForHWSIMD(var_types simdType, CorInfoType simdBaseJitType);
     CorInfoType getBaseJitTypeFromArgIfNeeded(NamedIntrinsic       intrinsic,
                                               CORINFO_CLASS_HANDLE clsHnd,
@@ -4497,6 +4499,10 @@ class Compiler
                               CorInfoType           simdBaseJitType,
                               var_types             retType,
                               unsigned              simdSize);
+    GenTree* impX86BaseIntrinsic(NamedIntrinsic        intrinsic,
+                                 CORINFO_METHOD_HANDLE method,
+                                 CORINFO_SIG_INFO*     sig,
+                                 CorInfoType           simdBaseJitType);
     GenTree* impSSEIntrinsic(NamedIntrinsic intrinsic, CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* sig);
     GenTree* impSSE2Intrinsic(NamedIntrinsic intrinsic, CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* sig);
     GenTree* impAvxOrAvx2Intrinsic(NamedIntrinsic intrinsic, CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* sig);

src/coreclr/jit/gentree.cpp

@@ -17956,6 +17956,8 @@ bool GenTree::isRMWHWIntrinsic(Compiler* comp)
         case NI_FMA_MultiplySubtractNegated:
         case NI_FMA_MultiplySubtractNegatedScalar:
         case NI_FMA_MultiplySubtractScalar:
+        case NI_X86Base_DivRem:
+        case NI_X86Base_X64_DivRem:
         {
             return true;
         }
@@ -21635,6 +21637,24 @@ GenTreeHWIntrinsic* Compiler::gtNewScalarHWIntrinsicNode(
                            /* isSimdAsHWIntrinsic */ false, op1, op2, op3);
 }
 
+GenTreeLclVar* Compiler::gtNewLclvForMultiRegIntrinsicNode(GenTreeHWIntrinsic* intrinsicNode, CORINFO_SIG_INFO* sig)
+{
+    assert(HWIntrinsicInfo::IsMultiReg(intrinsicNode->GetHWIntrinsicId()));
+
+    const unsigned lclNum = lvaGrabTemp(true DEBUGARG("Return value temp for multireg intrinsic"));
+    impAssignTempGen(lclNum, intrinsicNode, sig->retTypeSigClass, (unsigned)CHECK_SPILL_ALL);
+
+    LclVarDsc* varDsc = lvaGetDesc(lclNum);
+    // The following is to exclude the fields of the local to have SSA.
+    varDsc->lvIsMultiRegRet = true;
+
+    GenTreeLclVar* lclVar = gtNewLclvNode(lclNum, varDsc->lvType);
+    lclVar->SetDoNotCSE();
+    lclVar->SetMultiReg();
+
+    return lclVar;
+}
+
 // Returns true for the HW Intrinsic instructions that have MemoryLoad semantics, false otherwise
 bool GenTreeHWIntrinsic::OperIsMemoryLoad() const
 {

src/coreclr/jit/gentree.h

@@ -8130,6 +8130,7 @@ inline var_types GenTree::GetRegTypeByIndex(int regIndex) const
 #endif // !defined(TARGET_64BIT)
 #endif // FEATURE_MULTIREG_RET
 
+#ifdef FEATURE_HW_INTRINSICS
     if (OperIsHWIntrinsic())
     {
         assert(TypeGet() == TYP_STRUCT);
@@ -8146,9 +8147,10 @@ inline var_types GenTree::GetRegTypeByIndex(int regIndex) const
 #elif defined(TARGET_XARCH)
         // At this time, the only multi-reg HW intrinsics all return the type of their
         // arguments. If this changes, we will need a way to record or determine this.
-        return gtGetOp1()->TypeGet();
+        return AsHWIntrinsic()->Op(1)->TypeGet();
 #endif
     }
+#endif // FEATURE_HW_INTRINSICS
 
     if (OperIsScalarLocal())
     {

src/coreclr/jit/hwintrinsic.h

@@ -744,6 +744,12 @@ struct HWIntrinsicInfo
                 return 2;
 #endif
 
+#ifdef TARGET_XARCH
+            case NI_X86Base_DivRem:
+            case NI_X86Base_X64_DivRem:
+                return 2;
+#endif // TARGET_XARCH
+
             default:
                 unreached();
         }

src/coreclr/jit/hwintrinsicarm64.cpp

@@ -1582,27 +1582,15 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic        intrinsic,
                 }
             }
 
-            GenTree* loadIntrinsic = gtNewSimdHWIntrinsicNode(retType, op1, intrinsic, simdBaseJitType, simdSize);
+            GenTreeHWIntrinsic* loadIntrinsic =
+                gtNewSimdHWIntrinsicNode(retType, op1, intrinsic, simdBaseJitType, simdSize);
             // This operation contains an implicit indirection
             //   it could point into the global heap or
             //   it could throw a null reference exception.
             //
             loadIntrinsic->gtFlags |= (GTF_GLOB_REF | GTF_EXCEPT);
 
-            assert(HWIntrinsicInfo::IsMultiReg(intrinsic));
-
-            const unsigned lclNum = lvaGrabTemp(true DEBUGARG("Return value temp for multireg intrinsic"));
-            impAssignTempGen(lclNum, loadIntrinsic, sig->retTypeSigClass, (unsigned)CHECK_SPILL_ALL);
-
-            LclVarDsc* varDsc = lvaGetDesc(lclNum);
-            // The following is to exclude the fields of the local to have SSA.
-            varDsc->lvIsMultiRegRet = true;
-
-            GenTreeLclVar* lclVar = gtNewLclvNode(lclNum, varDsc->lvType);
-            lclVar->SetDoNotCSE();
-            lclVar->SetMultiReg();
-
-            retNode = lclVar;
+            retNode = gtNewLclvForMultiRegIntrinsicNode(loadIntrinsic, sig);
             break;
         }
 

src/coreclr/jit/hwintrinsiccodegenxarch.cpp

@@ -1175,6 +1175,43 @@ void CodeGen::genX86BaseIntrinsic(GenTreeHWIntrinsic* node)
             break;
         }
 
+        case NI_X86Base_DivRem:
+        case NI_X86Base_X64_DivRem:
+        {
+            assert(node->GetOperandCount() == 3);
+
+            // SIMD base type is from signature and can distinguish signed and unsigned
+            var_types   targetType = node->GetSimdBaseType();
+            GenTree*    op1        = node->Op(1);
+            GenTree*    op2        = node->Op(2);
+            GenTree*    op3        = node->Op(3);
+            instruction ins        = HWIntrinsicInfo::lookupIns(intrinsicId, targetType);
+
+            regNumber op1Reg = op1->GetRegNum();
+            regNumber op2Reg = op2->GetRegNum();
+            regNumber op3Reg = op3->GetRegNum();
+
+            emitAttr attr = emitTypeSize(targetType);
+            emitter* emit = GetEmitter();
+
+            // op1: EAX, op2: EDX, op3: free
+            assert(op1Reg != REG_EDX);
+            assert(op2Reg != REG_EAX);
+            if (op3->isUsedFromReg())
+            {
+                assert(op3Reg != REG_EDX);
+                assert(op3Reg != REG_EAX);
+            }
+
+            emit->emitIns_Mov(INS_mov, attr, REG_EAX, op1Reg, /* canSkip */ true);
+            emit->emitIns_Mov(INS_mov, attr, REG_EDX, op2Reg, /* canSkip */ true);
+
+            // emit the DIV/IDIV instruction
+            emit->emitInsBinary(ins, attr, node, op3);
+
+            break;
+        }
+
         default:
             unreached();
             break;

src/coreclr/jit/hwintrinsiclistxarch.h

@@ -224,6 +224,7 @@ HARDWARE_INTRINSIC(Vector256,       Xor,
 HARDWARE_INTRINSIC(X86Base,         BitScanForward,                             0,              1,      {INS_invalid,           INS_invalid,            INS_invalid,            INS_invalid,            INS_bsf,                INS_bsf,                INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid},           HW_Category_Scalar,                 HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics)
 HARDWARE_INTRINSIC(X86Base,         BitScanReverse,                             0,              1,      {INS_invalid,           INS_invalid,            INS_invalid,            INS_invalid,            INS_bsr,                INS_bsr,                INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid},           HW_Category_Scalar,                 HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics)
 HARDWARE_INTRINSIC(X86Base,         Pause,                                      0,              0,      {INS_invalid,           INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid},           HW_Category_Special,                HW_Flag_NoContainment|HW_Flag_NoRMWSemantics)
+HARDWARE_INTRINSIC(X86Base,         DivRem,                                     0,              3,      {INS_invalid,           INS_invalid,            INS_invalid,            INS_invalid,            INS_idiv,               INS_div,                INS_idiv,               INS_div,                INS_invalid,            INS_invalid},           HW_Category_Scalar,                 HW_Flag_NoFloatingPointUsed|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen)
 
 // ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
 //                 ISA              Function name                               SIMD size       NumArg                                                                                                         Instructions                                                                                                                             Category                            Flags
@@ -232,6 +233,7 @@ HARDWARE_INTRINSIC(X86Base,         Pause,
 //  X86Base 64-bit-only Intrinsics
 HARDWARE_INTRINSIC(X86Base_X64,     BitScanForward,                             0,              1,      {INS_invalid,           INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_bsf,                INS_bsf,                INS_invalid,            INS_invalid},           HW_Category_Scalar,                 HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics)
 HARDWARE_INTRINSIC(X86Base_X64,     BitScanReverse,                             0,              1,      {INS_invalid,           INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_bsr,                INS_bsr,                INS_invalid,            INS_invalid},           HW_Category_Scalar,                 HW_Flag_NoFloatingPointUsed|HW_Flag_NoRMWSemantics)
+HARDWARE_INTRINSIC(X86Base_X64,     DivRem,                                     0,              3,      {INS_invalid,           INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_invalid,            INS_idiv,               INS_div,                INS_invalid,            INS_invalid},           HW_Category_Scalar,                 HW_Flag_NoFloatingPointUsed|HW_Flag_BaseTypeFromSecondArg|HW_Flag_MultiReg|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen)
 
 // ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
 //                 ISA              Function name                               SIMD size       NumArg                                                                                                         Instructions                                                                                                                             Category                            Flags

src/coreclr/jit/hwintrinsicxarch.cpp

@@ -491,8 +491,10 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic        intrinsic,
     {
         case InstructionSet_Vector256:
         case InstructionSet_Vector128:
-        case InstructionSet_X86Base:
             return impBaseIntrinsic(intrinsic, clsHnd, method, sig, simdBaseJitType, retType, simdSize);
+        case InstructionSet_X86Base:
+        case InstructionSet_X86Base_X64:
+            return impX86BaseIntrinsic(intrinsic, method, sig, simdBaseJitType);
         case InstructionSet_SSE:
             return impSSEIntrinsic(intrinsic, method, sig);
         case InstructionSet_SSE2:
@@ -550,13 +552,8 @@ GenTree* Compiler::impBaseIntrinsic(NamedIntrinsic        intrinsic,
         return nullptr;
     }
 
-    var_types simdBaseType = TYP_UNKNOWN;
-
-    if (intrinsic != NI_X86Base_Pause)
-    {
-        simdBaseType = JitType2PreciseVarType(simdBaseJitType);
-        assert(varTypeIsArithmetic(simdBaseType));
-    }
+    var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType);
+    assert(varTypeIsArithmetic(simdBaseType));
 
     switch (intrinsic)
     {
@@ -2194,16 +2191,62 @@ GenTree* Compiler::impBaseIntrinsic(NamedIntrinsic        intrinsic,
             break;
         }
 
+        default:
+        {
+            return nullptr;
+        }
+    }
+
+    return retNode;
+}
+
+GenTree* Compiler::impX86BaseIntrinsic(NamedIntrinsic        intrinsic,
+                                       CORINFO_METHOD_HANDLE method,
+                                       CORINFO_SIG_INFO*     sig,
+                                       CorInfoType           simdBaseJitType)
+{
+    GenTree* retNode = nullptr;
+    GenTree* op1     = nullptr;
+    GenTree* op2     = nullptr;
+    GenTree* op3     = nullptr;
+    GenTree* op4     = nullptr;
+
+    var_types retType = JITtype2varType(sig->retType);
+
+    switch (intrinsic)
+    {
+
         case NI_X86Base_Pause:
         {
             assert(sig->numArgs == 0);
-            assert(JITtype2varType(sig->retType) == TYP_VOID);
-            assert(simdSize == 0);
+            assert(retType == TYP_VOID);
+            assert(simdBaseJitType == CORINFO_TYPE_UNDEF);
 
             retNode = gtNewScalarHWIntrinsicNode(TYP_VOID, intrinsic);
             break;
         }
 
+        case NI_X86Base_DivRem:
+        case NI_X86Base_X64_DivRem:
+        {
+            assert(sig->numArgs == 3);
+            assert(HWIntrinsicInfo::IsMultiReg(intrinsic));
+            assert(retType == TYP_STRUCT);
+            assert(simdBaseJitType != CORINFO_TYPE_UNDEF);
+
+            op3 = impPopStack().val;
+            op2 = impPopStack().val;
+            op1 = impPopStack().val;
+
+            GenTreeHWIntrinsic* divRemIntrinsic = gtNewScalarHWIntrinsicNode(retType, op1, op2, op3, intrinsic);
+
+            // Store the type from signature into SIMD base type for convenience
+            divRemIntrinsic->SetSimdBaseJitType(simdBaseJitType);
+
+            retNode = gtNewLclvForMultiRegIntrinsicNode(divRemIntrinsic, sig);
+            break;
+        }
+
         default:
         {
             return nullptr;

src/coreclr/jit/lower.cpp

@@ -6749,7 +6749,7 @@ bool Lowering::NodesAreEquivalentLeaves(GenTree* tree1, GenTree* tree2)
 bool Lowering::CheckMultiRegLclVar(GenTreeLclVar* lclNode, const ReturnTypeDesc* retTypeDesc)
 {
     bool canEnregister = false;
-#if FEATURE_MULTIREG_RET
+#if FEATURE_MULTIREG_RET || FEATURE_HW_INTRINSICS
     LclVarDsc* varDsc = comp->lvaGetDesc(lclNode->GetLclNum());
     if ((comp->lvaEnregMultiRegVars) && varDsc->lvPromoted)
     {
@@ -6776,10 +6776,10 @@ bool Lowering::CheckMultiRegLclVar(GenTreeLclVar* lclNode, const ReturnTypeDesc*
         }
     }
 #ifdef TARGET_XARCH
-    // For local stores on XARCH we only handle mismatched src/dest register count for
-    // calls of SIMD type. If the source was another lclVar similarly promoted, we would
+    // For local stores on XARCH we can't handle another lclVar source.
+    // If the source was another lclVar similarly promoted, we would
     // have broken it into multiple stores.
-    if (lclNode->OperIs(GT_STORE_LCL_VAR) && !lclNode->gtGetOp1()->OperIs(GT_CALL))
+    if (lclNode->OperIs(GT_STORE_LCL_VAR) && lclNode->gtGetOp1()->OperIs(GT_LCL_VAR))
     {
         canEnregister = false;
     }

src/coreclr/jit/lowerxarch.cpp

@@ -384,6 +384,16 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode)
 
             ContainBlockStoreAddress(blkNode, size, dstAddr);
         }
+        else if (src->OperIsHWIntrinsic())
+        {
+            assert(!blkNode->AsObj()->GetLayout()->HasGCPtr());
+            if (blkNode->OperIs(GT_STORE_OBJ))
+            {
+                blkNode->SetOper(GT_STORE_BLK);
+            }
+            blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll;
+            ContainBlockStoreAddress(blkNode, size, dstAddr);
+        }
         else
         {
             assert(blkNode->OperIs(GT_STORE_BLK, GT_STORE_DYN_BLK));
@@ -6478,6 +6488,20 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
                                 }
                                 break;
                             }
+                            case NI_X86Base_DivRem:
+                            case NI_X86Base_X64_DivRem:
+                            {
+                                // DIV only allows divisor (op3) in memory
+                                if (TryGetContainableHWIntrinsicOp(node, &op3, &supportsRegOptional))
+                                {
+                                    MakeSrcContained(node, op3);
+                                }
+                                else if (supportsRegOptional)
+                                {
+                                    op3->SetRegOptional();
+                                }
+                                break;
+                            }
 
                             default:
                             {

src/coreclr/jit/lsraxarch.cpp

@@ -2015,7 +2015,23 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
     }
 
     int srcCount = 0;
-    int dstCount = intrinsicTree->IsValue() ? 1 : 0;
+    int dstCount;
+
+    if (intrinsicTree->IsValue())
+    {
+        if (HWIntrinsicInfo::IsMultiReg(intrinsicId))
+        {
+            dstCount = HWIntrinsicInfo::GetMultiRegCount(intrinsicId);
+        }
+        else
+        {
+            dstCount = 1;
+        }
+    }
+    else
+    {
+        dstCount = 0;
+    }
 
     regMaskTP dstCandidates = RBM_NONE;
 
@@ -2193,6 +2209,26 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
             }
 #endif // TARGET_X86
 
+            case NI_X86Base_DivRem:
+            case NI_X86Base_X64_DivRem:
+            {
+                assert(numArgs == 3);
+                assert(dstCount == 2);
+                assert(isRMW);
+
+                // DIV implicitly put op1(lower) to EAX and op2(upper) to EDX
+                srcCount += BuildOperandUses(op1, RBM_EAX);
+                srcCount += BuildOperandUses(op2, RBM_EDX);
+                srcCount += op3->isContained() ? BuildOperandUses(op3) : BuildDelayFreeUses(op3, op1);
+
+                // result put in EAX and EDX
+                BuildDef(intrinsicTree, RBM_EAX, 0);
+                BuildDef(intrinsicTree, RBM_EDX, 1);
+
+                buildUses = false;
+                break;
+            }
+
             case NI_BMI2_MultiplyNoFlags:
             case NI_BMI2_X64_MultiplyNoFlags:
             {
@@ -2463,7 +2499,8 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
     }
     else
     {
-        assert(dstCount == 0);
+        // Currently dstCount = 2 is only used for DivRem, which has special constriants and handled above
+        assert((dstCount == 0) || (dstCount == 2));
     }
 
     *pDstCount = dstCount;