Skip to content

Commit 9c0eda6

Browse files
artagnonartagnon
committed
[VPlan] Use "hidden operand" approach
1 parent 6baba87 commit 9c0eda6

14 files changed

+91
-81
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7640,10 +7640,7 @@ VPWidenMemoryRecipe *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
76407640
Ptr, &Plan.getVF(), getLoadStoreType(I),
76417641
/*Stride*/ -1, Flags, VPI->getDebugLoc());
76427642
} else {
7643-
const DataLayout &DL = I->getDataLayout();
7644-
VPValue *Offset = Plan.getConstantInt(
7645-
DL.getIndexType(Ptr->getUnderlyingValue()->getType()), 0);
7646-
VectorPtr = new VPVectorPointerRecipe(Ptr, Offset, getLoadStoreType(I),
7643+
VectorPtr = new VPVectorPointerRecipe(Ptr, getLoadStoreType(I),
76477644
GEP ? GEP->getNoWrapFlags()
76487645
: GEPNoWrapFlags::none(),
76497646
VPI->getDebugLoc());

‎llvm/lib/Transforms/Vectorize/VPlan.h‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1971,27 +1971,25 @@ class VPVectorEndPointerRecipe : public VPRecipeWithIRFlags,
19711971
#endif
19721972
};
19731973

1974-
/// A recipe to compute the pointers for widened memory accesses of
1975-
/// SourceElementTy. The recipe produces a GEP when executed, but has different
1976-
/// semantics from a plain GEP; the pointer and initial offset (which should be
1977-
/// the constant zero, passed as ZeroOffset) are used while unrolling: for every
1978-
/// unrolled copy of the recipe, the offset is VF + the offset of the previous
1979-
/// copy. When no unrolling is in effect, this recipe can be optimized away, as
1980-
/// it is just a GEP x, 0.
1974+
/// A recipe to compute the pointers for widened memory accesses of \p
1975+
/// SourceElementTy. The recipe has one operand prior to unrolling, \p Ptr.
1976+
/// After unrolling, it has an extra operand which can be queried using \p
1977+
/// getOffset, and it produces `GEP Ptr, Offset` with flags \p GEPFlags, when
1978+
/// executed. The recipe is a no-op when no unrolling is in effect (IC = 1).
19811979
class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
19821980
Type *SourceElementTy;
19831981

19841982
public:
1985-
VPVectorPointerRecipe(VPValue *Ptr, VPValue *ZeroOffset,
1986-
Type *SourceElementTy, GEPNoWrapFlags GEPFlags,
1987-
DebugLoc DL)
1988-
: VPRecipeWithIRFlags(VPDef::VPVectorPointerSC, {Ptr, ZeroOffset},
1989-
GEPFlags, DL),
1983+
VPVectorPointerRecipe(VPValue *Ptr, Type *SourceElementTy,
1984+
GEPNoWrapFlags GEPFlags, DebugLoc DL)
1985+
: VPRecipeWithIRFlags(VPDef::VPVectorPointerSC, Ptr, GEPFlags, DL),
19901986
SourceElementTy(SourceElementTy) {}
19911987

19921988
VP_CLASSOF_IMPL(VPDef::VPVectorPointerSC)
19931989

1994-
VPValue *getOffset() { return getOperand(1); }
1990+
VPValue *getOffset() {
1991+
return getNumOperands() == 2 ? getOperand(1) : nullptr;
1992+
}
19951993

19961994
void execute(VPTransformState &State) override;
19971995

@@ -2012,9 +2010,11 @@ class VPVectorPointerRecipe : public VPRecipeWithIRFlags {
20122010
}
20132011

20142012
VPVectorPointerRecipe *clone() override {
2015-
return new VPVectorPointerRecipe(getOperand(0), getOffset(),
2016-
SourceElementTy, getGEPNoWrapFlags(),
2017-
getDebugLoc());
2013+
auto *Clone = new VPVectorPointerRecipe(getOperand(0), SourceElementTy,
2014+
getGEPNoWrapFlags(), getDebugLoc());
2015+
if (auto *Off = getOffset())
2016+
Clone->addOperand(Off);
2017+
return Clone;
20182018
}
20192019

20202020
/// Return the cost of this VPHeaderPHIRecipe.

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2505,6 +2505,8 @@ void VPVectorEndPointerRecipe::printRecipe(raw_ostream &O, const Twine &Indent,
25052505
void VPVectorPointerRecipe::execute(VPTransformState &State) {
25062506
auto &Builder = State.Builder;
25072507
Value *Ptr = State.get(getOperand(0), VPLane(0));
2508+
if (!getOffset())
2509+
return State.set(this, Ptr, /*IsScalar*/ true);
25082510
Value *Offset = State.get(getOffset(), true);
25092511
Value *ResultPtr = Builder.CreateGEP(getSourceElementType(), Ptr, Offset, "",
25102512
getGEPNoWrapFlags());

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1447,11 +1447,6 @@ static void simplifyRecipe(VPSingleDefRecipe *Def, VPTypeAnalysis &TypeInfo) {
14471447
}
14481448
}
14491449

1450-
// Simplify gep x, 0 -> x in unrolled VectorPointerRecipe.
1451-
if (auto *VPR = dyn_cast<VPVectorPointerRecipe>(Def))
1452-
if (match(VPR->getOffset(), m_ZeroInt()))
1453-
return VPR->replaceAllUsesWith(VPR->getOperand(0));
1454-
14551450
// VPScalarIVSteps for part 0 can be replaced by their start value, if only
14561451
// the first lane is demanded.
14571452
if (auto *Steps = dyn_cast<VPScalarIVStepsRecipe>(Def)) {

‎llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,14 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
300300
if (auto *VPR = dyn_cast<VPVectorPointerRecipe>(&R)) {
301301
VPBuilder Builder(VPR);
302302
auto *Prev = cast<VPVectorPointerRecipe>(getValueForPart(VPR, Part - 1))
303-
->getOperand(1);
303+
->getOffset();
304+
if (!Prev) {
305+
// Start with offset 0 for first part.
306+
const DataLayout &DL =
307+
Plan.getScalarHeader()->getIRBasicBlock()->getDataLayout();
308+
Prev = Plan.getConstantInt(
309+
DL.getIndexType(TypeInfo.inferScalarType(VPR)), 0);
310+
}
304311
VPValue *Increment = &Plan.getVF();
305312
Type *IncTy = TypeInfo.inferScalarType(Increment);
306313
Increment = Builder.createScalarZExtOrTrunc(
@@ -310,7 +317,7 @@ void UnrollState::unrollRecipeByUF(VPRecipeBase &R) {
310317
Builder.createOverflowingOp(Instruction::Add, {Prev, Increment},
311318
{true, true}, VPR->getDebugLoc());
312319
Copy->setOperand(0, VPR->getOperand(0));
313-
Copy->setOperand(1, Add);
320+
Copy->addOperand(Add);
314321
continue;
315322
}
316323
if (auto *Red = dyn_cast<VPReductionRecipe>(&R)) {

‎llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,10 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) "target-features"="+neon,+do
2626
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<[[ACC:%.+]]> = phi vp<[[RDX_START]]>, vp<[[REDUCE:%.+]]> (VF scaled by 1/4)
2727
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>, vp<[[VF]]>
2828
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<[[STEPS]]>
29-
; CHECK-NEXT: vp<[[PTR_A:%.+]]> = vector-pointer ir<%gep.a>, ir<0>
29+
; CHECK-NEXT: vp<[[PTR_A:%.+]]> = vector-pointer ir<%gep.a>
3030
; CHECK-NEXT: WIDEN ir<%load.a> = load vp<[[PTR_A]]>
3131
; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<[[STEPS]]>
32-
; CHECK-NEXT: vp<[[PTR_B:%.+]]> = vector-pointer ir<%gep.b>, ir<0>
32+
; CHECK-NEXT: vp<[[PTR_B:%.+]]> = vector-pointer ir<%gep.b>
3333
; CHECK-NEXT: WIDEN ir<%load.b> = load vp<[[PTR_B]]>
3434
; CHECK-NEXT: EXPRESSION vp<[[REDUCE]]> = ir<[[ACC]]> + partial.reduce.add (mul (ir<%load.b> zext to i32), (ir<%load.a> zext to i32))
3535
; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
@@ -83,9 +83,11 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) "target-features"="+neon,+do
8383
; CHECK-NEXT: EMIT-SCALAR vp<[[EP_IV:%.+]]> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
8484
; CHECK-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]> (VF scaled by 1/4)
8585
; CHECK-NEXT: CLONE ir<%gep.a> = getelementptr ir<%a>, vp<%index>
86-
; CHECK-NEXT: WIDEN ir<%load.a> = load ir<%gep.a>
86+
; CHECK-NEXT: vp<%2> = vector-pointer ir<%gep.a>
87+
; CHECK-NEXT: WIDEN ir<%load.a> = load vp<%2>
8788
; CHECK-NEXT: CLONE ir<%gep.b> = getelementptr ir<%b>, vp<%index>
88-
; CHECK-NEXT: WIDEN ir<%load.b> = load ir<%gep.b>
89+
; CHECK-NEXT: vp<%3> = vector-pointer ir<%gep.b>
90+
; CHECK-NEXT: WIDEN ir<%load.b> = load vp<%3>
8991
; CHECK-NEXT: WIDEN-CAST ir<%ext.b> = zext ir<%load.b> to i32
9092
; CHECK-NEXT: WIDEN-CAST ir<%ext.a> = zext ir<%load.a> to i32
9193
; CHECK-NEXT: WIDEN ir<%mul> = mul ir<%ext.b>, ir<%ext.a>

‎llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics-reduction.ll‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
4444
; IF-EVL-OUTLOOP-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
4545
; IF-EVL-OUTLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
4646
; IF-EVL-OUTLOOP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
47-
; IF-EVL-OUTLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>, ir<0>
47+
; IF-EVL-OUTLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>
4848
; IF-EVL-OUTLOOP-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
4949
; IF-EVL-OUTLOOP-NEXT: WIDEN ir<[[ADD:%.+]]> = add ir<[[LD1]]>, ir<[[RDX_PHI]]>
5050
; IF-EVL-OUTLOOP-NEXT: WIDEN-INTRINSIC vp<[[RDX_SELECT]]> = call llvm.vp.merge(ir<true>, ir<[[ADD]]>, ir<[[RDX_PHI]]>, vp<[[EVL]]>)
@@ -84,7 +84,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
8484
; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
8585
; IF-EVL-INLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
8686
; IF-EVL-INLOOP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
87-
; IF-EVL-INLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>, ir<0>
87+
; IF-EVL-INLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>
8888
; IF-EVL-INLOOP-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
8989
; IF-EVL-INLOOP-NEXT: REDUCE ir<[[ADD:%.+]]> = ir<[[RDX_PHI]]> + vp.reduce.add (ir<[[LD1]]>, vp<[[EVL]]>)
9090
; IF-EVL-INLOOP-NEXT: EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
@@ -121,7 +121,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
121121
; NO-VP-OUTLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
122122
; NO-VP-OUTLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>, vp<[[VF]]>
123123
; NO-VP-OUTLOOP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
124-
; NO-VP-OUTLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>, ir<0>
124+
; NO-VP-OUTLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>
125125
; NO-VP-OUTLOOP-NEXT: WIDEN ir<[[LD1:%.+]]> = load vp<[[PTR1]]>
126126
; NO-VP-OUTLOOP-NEXT: WIDEN ir<[[ADD:%.+]]> = add ir<[[LD1]]>, ir<[[RDX_PHI]]>
127127
; NO-VP-OUTLOOP-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add nuw vp<[[IV]]>, vp<[[VFUF]]>
@@ -169,7 +169,7 @@ define i32 @reduction(ptr %a, i64 %n, i32 %start) {
169169
; NO-VP-INLOOP-NEXT: WIDEN-REDUCTION-PHI ir<[[RDX_PHI:%.+]]> = phi vp<[[RDX_START]]>, ir<[[RDX_NEXT:%.+]]>
170170
; NO-VP-INLOOP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>, vp<[[VF]]>
171171
; NO-VP-INLOOP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
172-
; NO-VP-INLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>, ir<0>
172+
; NO-VP-INLOOP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>
173173
; NO-VP-INLOOP-NEXT: WIDEN ir<[[LD1:%.+]]> = load vp<[[PTR1]]>
174174
; NO-VP-INLOOP-NEXT: REDUCE ir<[[ADD:%.+]]> = ir<[[RDX_PHI]]> + reduce.add (ir<[[LD1]]>)
175175
; NO-VP-INLOOP-NEXT: EMIT vp<[[IV_NEXT_EXIT:%.+]]> = add nuw vp<[[IV]]>, vp<[[VFUF]]>

‎llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-intrinsics.ll‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,14 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
2828
; IF-EVL-NEXT: EMIT-SCALAR vp<[[EVL:%.+]]> = EXPLICIT-VECTOR-LENGTH vp<[[AVL]]>
2929
; IF-EVL-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[EVL_PHI]]>, ir<1>, vp<[[EVL]]>
3030
; IF-EVL-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
31-
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>, ir<0>
31+
; IF-EVL-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>
3232
; IF-EVL-NEXT: WIDEN ir<[[LD1:%.+]]> = vp.load vp<[[PTR1]]>, vp<[[EVL]]>
3333
; IF-EVL-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[ST]]>
34-
; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP2]]>, ir<0>
34+
; IF-EVL-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP2]]>
3535
; IF-EVL-NEXT: WIDEN ir<[[LD2:%.+]]> = vp.load vp<[[PTR2]]>, vp<[[EVL]]>
3636
; IF-EVL-NEXT: WIDEN ir<[[ADD:%.+]]> = add nsw ir<[[LD2]]>, ir<[[LD1]]>
3737
; IF-EVL-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
38-
; IF-EVL-NEXT: vp<[[PTR3:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP3]]>, ir<0>
38+
; IF-EVL-NEXT: vp<[[PTR3:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP3]]>
3939
; IF-EVL-NEXT: WIDEN vp.store vp<[[PTR3]]>, ir<[[ADD]]>, vp<[[EVL]]>
4040
; IF-EVL-NEXT: EMIT-SCALAR vp<[[CAST:%[0-9]+]]> = zext vp<[[EVL]]> to i64
4141
; IF-EVL-NEXT: EMIT vp<[[IV_NEXT]]> = add vp<[[CAST]]>, vp<[[EVL_PHI]]>
@@ -59,14 +59,14 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
5959
; NO-VP-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
6060
; NO-VP-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>, vp<[[VF]]>
6161
; NO-VP-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr inbounds ir<%b>, vp<[[ST]]>
62-
; NO-VP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>, ir<0>
62+
; NO-VP-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP1]]>
6363
; NO-VP-NEXT: WIDEN ir<[[LD1:%.+]]> = load vp<[[PTR1]]>
6464
; NO-VP-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr inbounds ir<%c>, vp<[[ST]]>
65-
; NO-VP-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP2]]>, ir<0>
65+
; NO-VP-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP2]]>
6666
; NO-VP-NEXT: WIDEN ir<[[LD2:%.+]]> = load vp<[[PTR2]]>
6767
; NO-VP-NEXT: WIDEN ir<[[ADD:%.+]]> = add nsw ir<[[LD2]]>, ir<[[LD1]]>
6868
; NO-VP-NEXT: CLONE ir<[[GEP3:%.+]]> = getelementptr inbounds ir<%a>, vp<[[ST]]>
69-
; NO-VP-NEXT: vp<[[PTR3:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP3]]>, ir<0>
69+
; NO-VP-NEXT: vp<[[PTR3:%[0-9]+]]> = vector-pointer inbounds ir<[[GEP3]]>
7070
; NO-VP-NEXT: WIDEN store vp<[[PTR3]]>, ir<[[ADD]]>
7171
; NO-VP-NEXT: EMIT vp<[[IV_NEXT:%.+]]> = add nuw vp<[[IV]]>, vp<[[VFUF]]>
7272
; NO-VP-NEXT: EMIT branch-on-count vp<[[IV_NEXT]]>, vp<[[VTC]]>
@@ -108,11 +108,11 @@ define void @safe_dep(ptr %p) {
108108
; CHECK-NEXT: EMIT vp<[[IV:%[0-9]+]]> = CANONICAL-INDUCTION
109109
; CHECK-NEXT: vp<[[ST:%[0-9]+]]> = SCALAR-STEPS vp<[[IV]]>, ir<1>, vp<[[VF]]>
110110
; CHECK-NEXT: CLONE ir<[[GEP1:%.+]]> = getelementptr ir<%p>, vp<[[ST]]>
111-
; CHECK-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>, ir<0>
111+
; CHECK-NEXT: vp<[[PTR1:%[0-9]+]]> = vector-pointer ir<[[GEP1]]>
112112
; CHECK-NEXT: WIDEN ir<[[V:%.+]]> = load vp<[[PTR1]]>
113113
; CHECK-NEXT: CLONE ir<[[OFFSET:.+]]> = add vp<[[ST]]>, ir<100>
114114
; CHECK-NEXT: CLONE ir<[[GEP2:%.+]]> = getelementptr ir<%p>, ir<[[OFFSET]]>
115-
; CHECK-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>, ir<0>
115+
; CHECK-NEXT: vp<[[PTR2:%[0-9]+]]> = vector-pointer ir<[[GEP2]]>
116116
; CHECK-NEXT: WIDEN store vp<[[PTR2]]>, ir<[[V]]>
117117
; CHECK-NEXT: EMIT vp<[[IV_NEXT:%.+]]> = add nuw vp<[[IV]]>, vp<[[VFUF]]>
118118
; CHECK-NEXT: EMIT branch-on-count vp<[[IV_NEXT]]>, vp<[[VTC]]>

‎llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,11 +42,11 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw
4242
; CHECK-NEXT: " EMIT vp\<[[CAN_IV:%.+]]\> = CANONICAL-INDUCTION ir\<0\>, vp\<[[CAN_IV_NEXT:%.+]]\>\l" +
4343
; CHECK-NEXT: " vp\<[[STEPS:%.+]]\> = SCALAR-STEPS vp\<[[CAN_IV]]\>, ir\<1\>, vp\<[[VF]]\>\l" +
4444
; CHECK-NEXT: " CLONE ir\<%arrayidx\> = getelementptr inbounds ir\<%y\>, vp\<[[STEPS]]\>\l" +
45-
; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer inbounds ir\<%arrayidx\>, ir\<0\>\l" +
45+
; CHECK-NEXT: " vp\<[[VEC_PTR:%.+]]\> = vector-pointer inbounds ir\<%arrayidx\>\l" +
4646
; CHECK-NEXT: " WIDEN ir\<%lv\> = load vp\<[[VEC_PTR]]\>\l" +
4747
; CHECK-NEXT: " WIDEN-INTRINSIC ir\<%call\> = call llvm.sqrt(ir\<%lv\>)\l" +
4848
; CHECK-NEXT: " CLONE ir\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" +
49-
; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer inbounds ir\<%arrayidx2\>, ir\<0\>\l" +
49+
; CHECK-NEXT: " vp\<[[VEC_PTR2:%.+]]\> = vector-pointer inbounds ir\<%arrayidx2\>\l" +
5050
; CHECK-NEXT: " WIDEN store vp\<[[VEC_PTR2]]\>, ir\<%call\>\l" +
5151
; CHECK-NEXT: " EMIT vp\<[[CAN_IV_NEXT]]\> = add nuw vp\<[[CAN_IV]]\>, vp\<[[VFxUF]]\>\l" +
5252
; CHECK-NEXT: " EMIT branch-on-count vp\<[[CAN_IV_NEXT]]\>, vp\<[[VEC_TC]]\>\l" +

‎llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) {
2121
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<[[VF]]>
2222
; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1>
2323
; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%dst>, vp<[[STEPS:%.+]]>
24-
; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer inbounds ir<%gep>, ir<0>
24+
; CHECK-NEXT: vp<[[VEC_PTR:%.+]]> = vector-pointer inbounds ir<%gep>
2525
; CHECK-NEXT: WIDEN store vp<[[VEC_PTR]]>, ir<%iv>
2626
; CHECK-NEXT: EMIT vp<[[CAN_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]>
2727
; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_INC]]>, vp<[[VEC_TC]]>
@@ -77,10 +77,10 @@ define void @iv_expand(ptr %p, i64 %n) {
7777
; CHECK-NEXT: ir<%iv> = WIDEN-INDUCTION ir<0>, ir<1>, vp<%0>
7878
; CHECK-NEXT: vp<%4> = SCALAR-STEPS vp<%3>, ir<1>
7979
; CHECK-NEXT: CLONE ir<%q> = getelementptr ir<%p>, vp<%4>
80-
; CHECK-NEXT: vp<%5> = vector-pointer ir<%q>, ir<0>
80+
; CHECK-NEXT: vp<%5> = vector-pointer ir<%q>
8181
; CHECK-NEXT: WIDEN ir<%x> = load vp<%5>
8282
; CHECK-NEXT: WIDEN ir<%y> = add ir<%x>, ir<%iv>
83-
; CHECK-NEXT: vp<%6> = vector-pointer ir<%q>, ir<0>
83+
; CHECK-NEXT: vp<%6> = vector-pointer ir<%q>
8484
; CHECK-NEXT: WIDEN store vp<%6>, ir<%y>
8585
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<%3>, vp<%1>
8686
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<%2>
@@ -110,9 +110,10 @@ define void @iv_expand(ptr %p, i64 %n) {
110110
; CHECK-NEXT: EMIT-SCALAR vp<[[SCALAR_PHI:%.+]]> = phi [ ir<0>, vector.ph ], [ vp<%index.next>, vector.body ]
111111
; CHECK-NEXT: WIDEN-PHI ir<%iv> = phi [ vp<[[INDUCTION]]>, vector.ph ], [ vp<%vec.ind.next>, vector.body ]
112112
; CHECK-NEXT: CLONE ir<%q> = getelementptr ir<%p>, vp<[[SCALAR_PHI]]>
113-
; CHECK-NEXT: WIDEN ir<%x> = load ir<%q>
113+
; CHECK-NEXT: vp<%8> = vector-pointer ir<%q>
114+
; CHECK-NEXT: WIDEN ir<%x> = load vp<%8>
114115
; CHECK-NEXT: WIDEN ir<%y> = add ir<%x>, ir<%iv>
115-
; CHECK-NEXT: WIDEN store ir<%q>, ir<%y>
116+
; CHECK-NEXT: WIDEN store vp<%8>, ir<%y>
116117
; CHECK-NEXT: EMIT vp<%index.next> = add nuw vp<[[SCALAR_PHI]]>, ir<8>
117118
; CHECK-NEXT: EMIT vp<%vec.ind.next> = add ir<%iv>, vp<[[BROADCAST_INC]]>
118119
; CHECK-NEXT: EMIT branch-on-count vp<%index.next>, vp<%n.vec>

0 commit comments

Comments
 (0)