@@ -1805,65 +1805,70 @@ Instruction *WidenIV::widenIVUse(WidenIV::NarrowIVDefUse DU, SCEVExpander &Rewri
18051805 return nullptr ;
18061806 }
18071807
1808- // Does this user itself evaluate to a recurrence after widening?
1809- WidenedRecTy WideAddRec = getExtendedOperandRecurrence (DU);
1810- if (!WideAddRec.first )
1811- WideAddRec = getWideRecurrence (DU);
1812-
1813- assert ((WideAddRec.first == nullptr ) ==
1814- (WideAddRec.second == ExtendKind::Unknown));
1815- if (!WideAddRec.first ) {
1816- // If use is a loop condition, try to promote the condition instead of
1817- // truncating the IV first.
1818- if (widenLoopCompare (DU))
1808+ auto tryAddRecExpansion = [&]() -> Instruction* {
1809+ // Does this user itself evaluate to a recurrence after widening?
1810+ WidenedRecTy WideAddRec = getExtendedOperandRecurrence (DU);
1811+ if (!WideAddRec.first )
1812+ WideAddRec = getWideRecurrence (DU);
1813+ assert ((WideAddRec.first == nullptr ) ==
1814+ (WideAddRec.second == ExtendKind::Unknown));
1815+ if (!WideAddRec.first )
18191816 return nullptr ;
18201817
1821- // We are here about to generate a truncate instruction that may hurt
1822- // performance because the scalar evolution expression computed earlier
1823- // in WideAddRec.first does not indicate a polynomial induction expression.
1824- // In that case, look at the operands of the use instruction to determine
1825- // if we can still widen the use instead of truncating its operand.
1826- if (widenWithVariantUse (DU))
1818+ // Reuse the IV increment that SCEVExpander created as long as it dominates
1819+ // NarrowUse.
1820+ Instruction *WideUse = nullptr ;
1821+ if (WideAddRec.first == WideIncExpr &&
1822+ Rewriter.hoistIVInc (WideInc, DU.NarrowUse ))
1823+ WideUse = WideInc;
1824+ else {
1825+ WideUse = cloneIVUser (DU, WideAddRec.first );
1826+ if (!WideUse)
1827+ return nullptr ;
1828+ }
1829+ // Evaluation of WideAddRec ensured that the narrow expression could be
1830+ // extended outside the loop without overflow. This suggests that the wide use
1831+ // evaluates to the same expression as the extended narrow use, but doesn't
1832+ // absolutely guarantee it. Hence the following failsafe check. In rare cases
1833+ // where it fails, we simply throw away the newly created wide use.
1834+ if (WideAddRec.first != SE->getSCEV (WideUse)) {
1835+ LLVM_DEBUG (dbgs () << " Wide use expression mismatch: " " : "
1836+ << *SE->getSCEV (WideUse) << " != " first
1837+ << " \n "
1838+ DeadInsts.emplace_back (WideUse);
18271839 return nullptr ;
1840+ };
18281841
1829- // This user does not evaluate to a recurrence after widening, so don't
1830- // follow it. Instead insert a Trunc to kill off the original use,
1831- // eventually isolating the original narrow IV so it can be removed.
1832- truncateIVUse (DU, DT, LI);
1833- return nullptr ;
1834- }
1842+ // if we reached this point then we are going to replace
1843+ // DU.NarrowUse with WideUse. Reattach DbgValue then.
1844+ replaceAllDbgUsesWith (*DU.NarrowUse , *WideUse, *WideUse, *DT);
18351845
1836- // Reuse the IV increment that SCEVExpander created as long as it dominates
1837- // NarrowUse.
1838- Instruction *WideUse = nullptr ;
1839- if (WideAddRec.first == WideIncExpr &&
1840- Rewriter.hoistIVInc (WideInc, DU.NarrowUse ))
1841- WideUse = WideInc;
1842- else {
1843- WideUse = cloneIVUser (DU, WideAddRec.first );
1844- if (!WideUse)
1845- return nullptr ;
1846- }
1847- // Evaluation of WideAddRec ensured that the narrow expression could be
1848- // extended outside the loop without overflow. This suggests that the wide use
1849- // evaluates to the same expression as the extended narrow use, but doesn't
1850- // absolutely guarantee it. Hence the following failsafe check. In rare cases
1851- // where it fails, we simply throw away the newly created wide use.
1852- if (WideAddRec.first != SE->getSCEV (WideUse)) {
1853- LLVM_DEBUG (dbgs () << " Wide use expression mismatch: " " : "
1854- << *SE->getSCEV (WideUse) << " != " first
1855- << " \n "
1856- DeadInsts.emplace_back (WideUse);
1846+ ExtendKindMap[DU.NarrowUse ] = WideAddRec.second ;
1847+ // Returning WideUse pushes it on the worklist.
1848+ return WideUse;
1849+ };
1850+
1851+ if (auto *I = tryAddRecExpansion ())
1852+ return I;
1853+
1854+ // If use is a loop condition, try to promote the condition instead of
1855+ // truncating the IV first.
1856+ if (widenLoopCompare (DU))
18571857 return nullptr ;
1858- }
18591858
1860- // if we reached this point then we are going to replace
1861- // DU.NarrowUse with WideUse. Reattach DbgValue then.
1862- replaceAllDbgUsesWith (*DU.NarrowUse , *WideUse, *WideUse, *DT);
1859+ // We are here about to generate a truncate instruction that may hurt
1860+ // performance because the scalar evolution expression computed earlier
1861+ // in WideAddRec.first does not indicate a polynomial induction expression.
1862+ // In that case, look at the operands of the use instruction to determine
1863+ // if we can still widen the use instead of truncating its operand.
1864+ if (widenWithVariantUse (DU))
1865+ return nullptr ;
18631866
1864- ExtendKindMap[DU.NarrowUse ] = WideAddRec.second ;
1865- // Returning WideUse pushes it on the worklist.
1866- return WideUse;
1867+ // This user does not evaluate to a recurrence after widening, so don't
1868+ // follow it. Instead insert a Trunc to kill off the original use,
1869+ // eventually isolating the original narrow IV so it can be removed.
1870+ truncateIVUse (DU, DT, LI);
1871+ return nullptr ;
18671872}
18681873
18691874// / Add eligible users of NarrowDef to NarrowIVUsers.
0 commit comments