[VPlan] Add verifier check for EVL, introduce m_SpecificSInt

Author: artagnon

llvm/include/llvm/ADT/APInt.h

@@ -561,6 +561,18 @@ class [[nodiscard]] APInt {
     return I1.zext(I2.getBitWidth()) == I2;
   }
 
+  /// Determine if two APInts have the same value, after sign-extending
+  /// one of them (if needed!) to ensure that the bit-widths match.
+  static bool isSameSignedValue(const APInt &I1, const APInt &I2) {
+    if (I1.getBitWidth() == I2.getBitWidth())
+      return I1 == I2;
+
+    if (I1.getBitWidth() > I2.getBitWidth())
+      return I1 == I2.sext(I1.getBitWidth());
+
+    return I1.sext(I2.getBitWidth()) == I2;
+  }
+
   /// Overload to compute a hash_code for an APInt value.
   LLVM_ABI friend hash_code hash_value(const APInt &Arg);
 

llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h

@@ -78,7 +78,10 @@ class VPBuilder {
     setInsertPoint(TheBB, IP);
   }
 
-  VPlan &getPlan() const { return *getInsertBlock()->getPlan(); }
+  VPlan &getPlan() const {
+    assert(getInsertBlock() && "Expected insert point to be set");
+    return *getInsertBlock()->getPlan();
+  }
 
   /// Clear the insertion point: created instructions will not be inserted into
   /// a block.
@@ -355,9 +358,8 @@ class VPBuilder {
         FPBinOp ? FPBinOp->getFastMathFlags() : FastMathFlags(), DL));
   }
 
-  /// Create and insert a VectorEndPointerRecipe, with an optional \p
-  /// VFReplacement parameter, which is used in place of the VF recipe: useful
-  /// when there is an EVL recipe instead.
+  /// Create and insert a VectorEndPointerRecipe: requires insert-point to be
+  /// set.
   VPVectorEndPointerRecipe *
   createVectorEndPointerRecipe(VPValue *Ptr, Type *SourceElementType,
                                int64_t Stride, GEPNoWrapFlags GEPFlags,

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -8313,7 +8313,7 @@ VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
 }
 
 VPVectorEndPointerRecipe *
-VPBuilder::createVectorEndPointerRecipe(VPValue *Ptr, Type *SourceElementType,
+VPBuilder::createVectorEndPointerRecipe(VPValue *Ptr, Type *SourceElementTy,
                                         int64_t Stride, GEPNoWrapFlags GEPFlags,
                                         VPValue *VF, DebugLoc DbgLoc) {
   // Offset for Part 0 = Stride * (VF - 1).
@@ -8325,14 +8325,14 @@ VPBuilder::createVectorEndPointerRecipe(VPValue *Ptr, Type *SourceElementType,
   Type *VFTy = TypeInfo.inferScalarType(VF);
   VPValue *VFCast =
       createScalarZExtOrTrunc(VF, IndexTy, VFTy, DebugLoc::getUnknown());
-  VPValue *VFMinusOne = createOverflowingOp(
+  VPInstruction *VFMinusOne = createOverflowingOp(
       Instruction::Sub, {VFCast, Plan.getConstantInt(IndexTy, 1u)},
       {true, true});
-  VPValue *StridexVFMinusOne = createOverflowingOp(
+  VPInstruction *StridexVFMinusOne = createOverflowingOp(
       Instruction::Mul,
       {VFMinusOne, Plan.getConstantInt(IndexTy, Stride, /*IsSigned=*/true)});
   auto *VEPR = tryInsertInstruction(new VPVectorEndPointerRecipe(
-      Ptr, StridexVFMinusOne, SourceElementType, Stride, GEPFlags, DbgLoc));
+      Ptr, StridexVFMinusOne, SourceElementTy, Stride, GEPFlags, DbgLoc));
   return VEPR;
 }
 

llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h

@@ -93,10 +93,9 @@ struct deferredval_ty {
 /// whichever value m_VPValue(X) populated.
 inline deferredval_ty m_Deferred(VPValue *const &V) { return V; }
 
-/// Match an integer constant or vector of constants if Pred::isValue returns
-/// true for the APInt. \p BitWidth optionally specifies the bitwidth the
-/// matched constant must have. If it is 0, the matched constant can have any
-/// bitwidth.
+/// Match an integer constant if Pred::isValue returns true for the APInt. \p
+/// BitWidth optionally specifies the bitwidth the matched constant must have.
+/// If it is 0, the matched constant can have any bitwidth.
 template <typename Pred, unsigned BitWidth = 0> struct int_pred_ty {
   Pred P;
 
@@ -117,15 +116,18 @@ template <typename Pred, unsigned BitWidth = 0> struct int_pred_ty {
   }
 };
 
-/// Match a specified integer value or vector of all elements of that
-/// value. \p BitWidth optionally specifies the bitwidth the matched constant
-/// must have. If it is 0, the matched constant can have any bitwidth.
+/// Match a specified signed or unsigned integer value.
 struct is_specific_int {
   APInt Val;
+  bool IsSigned;
 
-  is_specific_int(APInt Val) : Val(std::move(Val)) {}
+  is_specific_int(APInt Val, bool IsSigned = false)
+      : Val(std::move(Val)), IsSigned(IsSigned) {}
 
-  bool isValue(const APInt &C) const { return APInt::isSameValue(Val, C); }
+  bool isValue(const APInt &C) const {
+    return IsSigned ? APInt::isSameSignedValue(Val, C)
+                    : APInt::isSameValue(Val, C);
+  }
 };
 
 template <unsigned Bitwidth = 0>
@@ -135,6 +137,11 @@ inline specific_intval<0> m_SpecificInt(uint64_t V) {
   return specific_intval<0>(is_specific_int(APInt(64, V)));
 }
 
+inline specific_intval<0> m_SpecificSInt(int64_t V) {
+  return specific_intval<0>(
+      is_specific_int(APInt(64, V, /*isSigned=*/true), /*IsSigned=*/true));
+}
+
 inline specific_intval<1> m_False() {
   return specific_intval<1>(is_specific_int(APInt(64, 0)));
 }

llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

@@ -2931,11 +2931,11 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
         VEPR->getGEPNoWrapFlags(), &EVL, VEPR->getDebugLoc());
   };
 
-  auto m_VecEndPtrVF = [&Plan](VPValue *&Addr) { // NOLINT
+  auto m_VecEndPtrVF = [&Plan](VPValue *&Addr, int64_t Stride) { // NOLINT
     return m_VecEndPtr(
         m_VPValue(Addr),
         m_c_Mul(
-            m_VPValue(),
+            m_SpecificSInt(Stride),
             m_Sub(m_ZExtOrTruncOrSelf(m_Specific(&Plan->getVF())), m_One())));
   };
 
@@ -2949,7 +2949,10 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
   if (match(&CurRecipe, m_Reverse(m_VPValue(ReversedVal))) &&
       match(ReversedVal,
             m_MaskedLoad(m_VPValue(EndPtr), m_RemoveMask(HeaderMask, Mask))) &&
-      match(EndPtr, m_VecEndPtrVF(Addr)) &&
+      isa<VPVectorEndPointerRecipe>(EndPtr) &&
+      match(EndPtr,
+            m_VecEndPtrVF(
+                Addr, cast<VPVectorEndPointerRecipe>(EndPtr)->getStride())) &&
       cast<VPWidenLoadRecipe>(ReversedVal)->isReverse()) {
     auto *LoadR = new VPWidenLoadEVLRecipe(
         *cast<VPWidenLoadRecipe>(ReversedVal), AdjustEndPtr(EndPtr), EVL, Mask);
@@ -2969,7 +2972,10 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask,
   if (match(&CurRecipe,
             m_MaskedStore(m_VPValue(EndPtr), m_Reverse(m_VPValue(ReversedVal)),
                           m_RemoveMask(HeaderMask, Mask))) &&
-      match(EndPtr, m_VecEndPtrVF(Addr)) &&
+      isa<VPVectorEndPointerRecipe>(EndPtr) &&
+      match(EndPtr,
+            m_VecEndPtrVF(
+                Addr, cast<VPVectorEndPointerRecipe>(EndPtr)->getStride())) &&
       cast<VPWidenStoreRecipe>(CurRecipe).isReverse()) {
     auto *NewReverse = new VPWidenIntrinsicRecipe(
         Intrinsic::experimental_vp_reverse,
@@ -3557,10 +3563,9 @@ void VPlanTransforms::createInterleaveGroups(
     // the pointer operand of the interleaved access is supposed to be uniform.
     if (IG->isReverse()) {
       B.setInsertPoint(InsertPos);
-      auto *ReversePtr = B.createVectorEndPointerRecipe(
+      Addr = B.createVectorEndPointerRecipe(
           Addr, getLoadStoreType(IRInsertPos), -(int64_t)IG->getFactor(), NW,
           &Plan.getVF(), InsertPos->getDebugLoc());
-      Addr = ReversePtr;
     }
     auto *VPIG = new VPInterleaveRecipe(IG, Addr, StoredValues,
                                         InsertPos->getMask(), NeedsMaskForGaps,

llvm/lib/Transforms/Vectorize/VPlanVerifier.cpp

@@ -156,7 +156,17 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
     }
     return true;
   };
-  return all_of(EVL.users(), [this, &VerifyEVLUse](VPUser *U) {
+  auto VerifyEVLUseInVecEndPtr = [&EVL](auto &VEPRs) {
+    if (all_of(VEPRs, [&EVL](VPUser *U) {
+          auto *VEPR = cast<VPVectorEndPointerRecipe>(U);
+          return match(VEPR->getOffset(),
+                       m_c_Mul(m_VPValue(), m_Sub(m_Specific(&EVL), m_One())));
+        }))
+      return true;
+    errs() << "Expected VectorEndPointer with EVL operand\n";
+    return false;
+  };
+  return all_of(EVL.users(), [&](VPUser *U) {
     return TypeSwitch<const VPUser *, bool>(U)
         .Case<VPWidenIntrinsicRecipe>([&](const VPWidenIntrinsicRecipe *S) {
           return VerifyEVLUse(*S, S->getNumOperands() - 1);
@@ -179,8 +189,17 @@ bool VPlanVerifier::verifyEVLRecipe(const VPInstruction &EVL) const {
           if (I->getOpcode() == Instruction::PHI ||
               I->getOpcode() == Instruction::ICmp)
             return VerifyEVLUse(*I, 1);
-          if (I->getOpcode() == Instruction::Sub)
-            return true;
+          if (I->getOpcode() == Instruction::Sub) {
+            auto *VPI =
+                dyn_cast_if_present<VPSingleDefRecipe>(I->getSingleUser());
+            if (VPI) {
+              auto VEPRs = make_filter_range(VPI->users(),
+                                             IsaPred<VPVectorEndPointerRecipe>);
+              if (!VEPRs.empty())
+                return VerifyEVLUseInVecEndPtr(VEPRs);
+            }
+            return VerifyEVLUse(*I, 1);
+          }
           switch (I->getOpcode()) {
           case Instruction::Add:
             break;